Skip to content

Commit 8b0bdcd

Browse files
NicolappsConvex, Inc
Nicolapps
authored and
Convex, Inc
committed
Publish the Fivetran crates in convex-backend instead of standalone repos (#27221)
GitOrigin-RevId: 4b5a9344ba51772d350bf2b5303fc8b616a308b4
1 parent d7d174f commit 8b0bdcd

File tree

15 files changed

+2098
-1
lines changed

15 files changed

+2098
-1
lines changed

‎Cargo.lock

Lines changed: 31 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[workspace]
22
members = [ "crates/*", "crates/convex/sync_types" ]
33
resolver = "2"
4-
exclude = [ "crates/fivetran_source", "crates/py_client", "crates/python_client_tests" ]
4+
exclude = [ "crates/py_client", "crates/python_client_tests" ]
55

66
[workspace.dependencies]
77
aes = { version = "0.8.4" }

‎crates/fivetran_destination/README.md

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Fivetran Destination Connector
2+
3+
This crate contains a destination connector allowing developers using Convex to
4+
replicate the data they have in other data sources to their Convex deployments.
5+
6+
The connector consists of a gRPC server hosted on the Fivetran infrastructure.
7+
It communicates with the Convex deployment using the HTTP API it provides.
8+
9+
## Installation
10+
11+
Make sure you have Git and Cargo installed. We recommend installing Cargo via
12+
[rustup](https://rustup.rs/).
13+
14+
```
15+
git clone https://github.com/get-convex/convex-backend.git
16+
cd convex-backend
17+
cargo build --release -p convex_fivetran_destination
18+
```
19+
20+
You can then find the executable file in
21+
`convex-backend/target/release/convex_fivetran_destination`.
22+
23+
## Usage
24+
25+
You can start the connector by starting its binary:
26+
27+
```
28+
$ ./convex_fivetran_destination
29+
{"level":"INFO","message":"Starting the destination on 0.0.0.0:50052","message-origin":"sdk_destination"}
30+
```
31+
32+
You can change the port used using the optional `--port` parameter:
33+
34+
```
35+
$ ./convex_fivetran_destination --port 1337
36+
{"level":"INFO","message":"Starting the destination on 0.0.0.0:1337","message-origin":"sdk_destination"}
37+
```
38+
39+
## Implementation
40+
41+
Unlike the
42+
[Fivetran source connector](https://github.com/get-convex/convex-backend/tree/main/crates/fivetran_source),
43+
the destination connector does not manage the state of the synchronization
44+
mechanism. This is done by Fivetran, which will call the relevant gRPC API
45+
endpoints depending on the state of the data source.

‎crates/fivetran_source/CHANGELOG.md

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Upcoming
2+
3+
# 0.6.0
4+
5+
- Update rust nightly version in .rust-toolchain
6+
- Remove deprecated set/map support
7+
- Add gzip support to connector
8+
9+
# 0.5.0
10+
11+
- Emit a TRUNCATE op before the first write to any given table (in case it's a
12+
resync)
13+
- Skip emitting an initial sync snapshot before the first page.
14+
15+
# 0.4.0
16+
17+
- Revert the `_convex_cursor` changes.
18+
- Few updates to loglines
19+
- docs updates
20+
21+
# 0.3.0
22+
23+
- Support a `_convex_cursor` table with a single column `cursor` which holds the
24+
`document_deltas` cursor representing the most recent sync.
25+
- Add documentation to docs/
26+
27+
# 0.2.0
28+
29+
- Bump `convex` dep to 0.5.0
30+
- Use /test_streaming_export_connection and /get_tables_and_columns endpoints
31+
rather than json_schemas json_schemas has stricter requirements around nested
32+
schemas than what fivetran requires.

‎crates/fivetran_source/Cargo.toml

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
[package]
2+
name = "convex_fivetran_source"
3+
description = "Fivetran source connector for Convex (convex.dev)"
4+
version = "0.6.0"
5+
authors = ["Convex, Inc. <[email protected]>"]
6+
edition = "2021"
7+
resolver = "2"
8+
license = "Apache-2.0"
9+
repository = "https://github.com/get-convex/convex-fivetran-source"
10+
homepage = "https://www.convex.dev/"
11+
12+
[dependencies]
13+
anyhow = { workspace = true }
14+
async-trait = { workspace = true }
15+
clap = { workspace = true, features = ["derive"] }
16+
convex = { path = "../convex", features = ["native-tls-vendored"] }
17+
convex_fivetran_common = { path = "../fivetran_common" }
18+
derive_more = { workspace = true }
19+
futures = { workspace = true }
20+
futures-async-stream = { workspace = true }
21+
maplit = { workspace = true }
22+
prost = { workspace = true }
23+
prost-types = { workspace = true }
24+
reqwest = { workspace = true, features = ["json", "native-tls-vendored"] }
25+
serde = { workspace = true, features = ["derive"] }
26+
serde_json = { workspace = true }
27+
tokio = { workspace = true }
28+
tonic = { workspace = true, features = ["gzip"] }
29+
30+
[build-dependencies]
31+
bytes = { workspace = true }
32+
cfg-if = { workspace = true }
33+
futures-util = { workspace = true }
34+
reqwest = { workspace = true, features = ["native-tls-vendored"] }
35+
tokio = { workspace = true }
36+
tonic-build = { workspace = true }
37+
38+
[dev-dependencies]
39+
convex = { path = "../convex", features = ["testing"] }
40+
convex_fivetran_common = { path = "../fivetran_common", features = ["testing"] }
41+
proptest = { workspace = true }
42+
proptest-derive = { workspace = true }
43+
rand = { workspace = true }
44+
schemars = { version = "0.8" }
45+
uuid = { workspace = true }
46+
47+
[package.metadata.cargo-machete]
48+
ignored = [
49+
# Build dependencies not understood
50+
"bytes",
51+
"cfg_if",
52+
"futures_util",
53+
"tonic_build",
54+
# Prost required via tonic macro
55+
"prost",
56+
]

‎crates/fivetran_source/README.md

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Fivetran Source Connector
2+
3+
This crate contains a source connector allowing developers using Convex to
4+
replicate the data they store in Convex to other databases.
5+
6+
The connector consists of a gRPC server hosted on the Fivetran infrastructure.
7+
This server retrieves the data it needs using the HTTP API described
8+
[in the Convex docs](https://docs.convex.dev/http-api/).
9+
10+
## Installation
11+
12+
Make sure you have Git and Cargo installed. We recommend installing Cargo via
13+
[rustup](https://rustup.rs/).
14+
15+
```
16+
git clone https://github.com/get-convex/convex-backend.git
17+
cd convex-backend
18+
cargo build --release -p convex_fivetran_source
19+
```
20+
21+
You can then find the executable file in
22+
`convex-backend/target/release/convex_fivetran_source`.
23+
24+
## Usage
25+
26+
You can start the connector by starting its binary:
27+
28+
```
29+
$ ./convex_fivetran_source
30+
Starting the connector on [::]:50051
31+
```
32+
33+
You can change the port used using the optional `--port` parameter:
34+
35+
```
36+
$ ./convex_fivetran_source --port 1337
37+
Starting the connector on [::]:1337
38+
```
39+
40+
## Sync Mechanism
41+
42+
The data synchronization happens in two steps:
43+
44+
- During the initial synchronization, the connector uses the
45+
[`list_snapshot`](https://docs.convex.dev/http-api/#get-apilist_snapshot) API
46+
to copy all documents.
47+
- During subsequent synchronizations, the connector uses the
48+
[`document_deltas`](https://docs.convex.dev/http-api/#get-apidocument_deltas)
49+
API to only apply changes from documents that were modified since the last
50+
synchronization.
51+
52+
![Flowchart showing the synchronization mechanism.](flow.png)

‎crates/fivetran_source/docs/overview.md

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
---
2+
name: Convex
3+
title: Convex connector for Fivetran
4+
description: Documentation and setup guide for the Convex connector for Fivetran
5+
---
6+
7+
# Convex {% typeBadge connector="convex" /%} {% availabilityBadge connector="convex" /%}
8+
9+
[Convex](https://convex.dev) is a full-stack TypeScript development platform.
10+
Replace your database, server functions, and glue code.
11+
12+
---
13+
14+
## Setup guide
15+
16+
Follow our [step-by-step Convex setup guide](/docs/databases/convex/setup-guide) to connect your Convex database with Fivetran.
17+
18+
---
19+
20+
## Sync overview
21+
22+
Once Fivetran is connected to your Convex deployment, the connector fetches an initial consistent snapshot of all data from your Convex database. Once the initial sync is complete, the connector uses CDC to efficiently incrementally sync updates at a newer consistent view of your Convex deployment. You can configure the frequency of these updates.
23+
24+
---
25+
26+
## Configuration
27+
28+
You will need your deployment URL and deploy key in order to configure the Convex Connector for Fivetran. You can find both on your project's [Production Deployment Settings page](https://docs.convex.dev/dashboard/deployments/deployment-settings).
29+
30+
---
31+
32+
## Schema information
33+
34+
Fivetran tries to replicate the database and columns from your configured Convex deployment to your destination according to Fivetran's [standard database update strategies](/docs/databases#transformationandmappingoverview).
35+
36+
### Type transformations and mapping
37+
38+
As the connector extracts your data, it matches [Convex data types](https://docs.convex.dev/database/types) to types that Fivetran supports.
39+
40+
The following table illustrates how the connector transforms your Convex data types into Fivetran-supported types:
41+
42+
| Convex Type | Fivetran Type | Fivetran Supported |
43+
| ----------- | ------------- | ------------------ |
44+
| Id | STRING | True |
45+
| Null | NULL | True |
46+
| Int64 | LONG | True |
47+
| Float64 | DOUBLE | True |
48+
| Boolean | BOOLEAN | True |
49+
| String | STRING | True |
50+
| Bytes | BINARY | True |
51+
| Array | JSON | True |
52+
| Object | JSON | True |
53+
54+
> NOTE: The `_creationTime` system field in each document is special-cased to convert into a UTC_DATETIME, despite being stored as a Float64 inside of Convex.
55+
56+
> NOTE: Nested types inside Object and Array are serialized as JSON using the [JSON format for export](https://docs.convex.dev/database/types).
57+
58+
### Nested data
59+
60+
Convex documents are represented as JSON [by using conversions](https://docs.convex.dev/database/types). If the first-level field is a simple data type, the connector will map it to its own type. If it's a complex nested data type such as an array or JSON data, it maps to a JSON type without unpacking. The connector does not automatically unpack nested JSON objects to separate tables in the destination. Any nested JSON objects are preserved as is in the destination so that you can use JSON processing functions.
61+
62+
For example, the following Convex document:
63+
64+
```json
65+
{"street" : "Main St."
66+
"city" : "New York"
67+
"country" : "US"
68+
"phone" : "(555) 123-5555"
69+
"zip code" : 12345
70+
"people" : ["John", "Jane", "Adam"]
71+
"car" : {"make" : "Honda",
72+
"year" : 2014,
73+
"type" : "AWD"}
74+
}
75+
```
76+
77+
is converted to the following table when the connector loads it into your destination:
78+
79+
| \_id | street | city | country | phone | zip code | people | car |
80+
| ---- | -------- | -------- | ------- | -------------- | -------- | ------------------------ | ------------------------------------------------- |
81+
| 1 | Main St. | New York | US | (555) 123-5555 | 12345 | ["John", "Jane", "Adam"] | {"make" : "Honda", "year" : 2014, "type" : "AWD"} |
82+
83+
### Fivetran-generated data
84+
85+
Fivetran adds the following column to every table in your destination:
86+
87+
- `_fivetran_synced` (UTC TIMESTAMP) indicates the time when Fivetran last successfully synced the row. It is added to every table.
88+
- `_fivetran_deleted` (BOOLEAN) indicates if the column was deleted in the source.
89+
90+
Fivetran adds these columns to give you insight into the state of your data and the progress of your data syncs.

‎crates/fivetran_source/docs/setup-guide.md

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
---
2+
name: Convex Setup Guide
3+
title: Convex Connector Setup Guide
4+
description: Read step-by-step instructions on how to connect your Convex deployment with your destination using Fivetran connectors.
5+
menuPosition: 0
6+
---
7+
8+
​
9+
10+
# Convex Setup Guide {% typeBadge connector="convex" /%} {% availabilityBadge connector="convex" /%}
11+
12+
​ Follow our setup guide to connect Convex to Fivetran. ​
13+
14+
---
15+
16+
​
17+
18+
## Prerequisites
19+
20+
​To connect your Convex deployment to Fivetran, you need the following:
21+
- A Convex account on a [Professional plan](https://www.convex.dev/plans)
22+
- A Convex deployment. See [Convex's documentation](https://docs.convex.dev/) to get started.
23+
- Your Convex deployment's URL (e.g., `https://jaded-raven-991.convex.cloud`)
24+
- Your Convex deployment's deploy key. You can find both the deployment URL and deploy key on the [Production Deployment Settings](https://docs.convex.dev/dashboard/deployments/deployment-settings) page. ​
25+
26+
---
27+
28+
​
29+
30+
## Setup instructions
31+
32+
​
33+
34+
> IMPORTANT: You must have a [Convex Professional plan](https://www.convex.dev/plans) to be able use the Convex connector. ​
35+
36+
### <span class="step-item">Locate your Deployment Credentials</span>
37+
38+
1. Navigate to your deployment on the [Convex Dashboard](https://dashboard.convex.dev/).​
39+
2. Navigate to the [Production Deployment Settings](https://docs.convex.dev/dashboard/deployments/deployment-settings).
40+
3. Find your deployment URL and deploy key and make a note of them. You will need them to configure Fivetran.
41+
42+
### <span class="step-item">Finish Fivetran configuration</span>
43+
44+
1. In your [connector setup form](/docs/getting-started/fivetran-dashboard/connectors#addanewconnector), enter a destination schema prefix. This prefix applies to each replicated schema and cannot be changed once your connector is created. ​
45+
2. Select Convex as your source connector.
46+
3. Enter your deployment credentials.
47+
4. Click **Save & Test**. Fivetran tests and validates our connection to your Convex deployment. Upon successful completion of the setup tests, you can sync your data using Fivetran. ​
48+
49+
### Setup tests
50+
51+
Fivetran performs the following tests to ensure that we can connect to your Convex deployment.
52+
53+
- Validating that your deployment credentials.
54+
- Ensuring you are on a [Convex Professional plan](https://www.convex.dev/plans).
55+
56+
---
57+
58+
## Related articles
59+
60+
​
61+
[<i aria-hidden="true" class="material-icons">description</i> Connector Overview](/docs/databases/convex)
62+
​ <b> </b> ​
63+
[<i aria-hidden="true" class="material-icons">home</i> Documentation Home](/docs/getting-started)

‎crates/fivetran_source/flow.png

170 KB
Loading

0 commit comments

Comments
 (0)