Skip to content

Commit 40c5e04

Browse files
viragtripathiviragtripathi
committed
updated redis-connect-postgres artifacts
1 parent f3f4f08 commit 40c5e04

File tree

15 files changed

+220
-145
lines changed

15 files changed

+220
-145
lines changed

‎connectors/postgres/README.md

Lines changed: 48 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -212,26 +212,30 @@ If you encounter <a href="https://debezium.io/documentation/reference/stable/con
212212

213213
```yml
214214
connections:
215-
- id: jobConfigConnection
215+
- id: jobManagerConnection #Redis Connect Job Metadata connection
216216
type: Redis
217-
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
218-
- id: targetConnection
217+
url: redis://127.0.0.1:14001 #this is based on lettuce uri syntax
218+
jobmanager.username: ${REDISCONNECT_JOBMANAGER_USERNAME} #this can be overridden by an env variable or a property file
219+
jobmanager.password: ${REDISCONNECT_JOBMANAGER_PASSWORD} #this can be overridden by an env variable or a property file
220+
#credentials.file.path: <path to <redisconnect_credentials_jobmanager_<job_name> e.g. /var/secrets/jobmanager> when username and password are not provided here
221+
- id: targetConnection #target Redis connection
219222
type: Redis
220-
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14000
221-
- id: metricsConnection
222-
type: Redis
223-
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
223+
url: redis://127.0.0.1:14000 #this is based on lettuce uri syntax
224+
target.username: ${REDISCONNECT_TARGET_USERNAME} #this can be overridden by an env variable or a property file
225+
target.password: ${REDISCONNECT_TARGET_PASSWORD} #this can be overridden by an env variable or a property file
226+
#credentials.file.path: <path to <redisconnect_credentials_redis_<job_name> e.g. /var/secrets/redis> when username and password are not provided here
224227
- id: RDBConnection
225228
type: RDB
226229
name: RedisConnect #database pool name
227230
database: RedisConnect #database
228-
url: "jdbc:postgresql://127.0.0.1:5432/RedisConnect"
231+
url: "jdbc:postgresql://127.0.0.1:5432/RedisConnect" #this is jdbc client driver specific, and it can contain any supported parameters
229232
host: 127.0.0.1
230233
port: 5432
231-
username: ${REDISCONNECT_SOURCE_USERNAME}
232-
password: ${REDISCONNECT_SOURCE_PASSWORD}
233-
#heartbeat.interval.ms: 10000
234-
#heartbeat.action.query: "INSERT INTO heartbeat (id, ts) VALUES (1, NOW()) ON CONFLICT(id) DO UPDATE SET ts=EXCLUDED.ts;"
234+
source.username: ${REDISCONNECT_SOURCE_USERNAME} #this can be overridden by an env variable or a property file
235+
source.password: ${REDISCONNECT_SOURCE_PASSWORD} #this can be overridden by an env variable or a property file
236+
#credentials.file.path: <path to <redisconnect_credentials_postgresql_<job_name> e.g. /var/secrets/postgresql> when username and password are not provided here
237+
#heartbeat.interval.ms: 10000 #Workaround for AWS RDS PG WAL space issue
238+
#heartbeat.action.query: "INSERT INTO heartbeat (id, ts) VALUES (1, NOW()) ON CONFLICT(id) DO UPDATE SET ts=EXCLUDED.ts;" #Workaround for AWS RDS PG WAL space issue
235239
```
236240

237241
</p>
@@ -245,37 +249,37 @@ connections:
245249
### Sample Setup.yml under redis-connect-postgres/config/samples/postgres folder
246250

247251
```yml
248-
connectionId: jobConfigConnection
252+
connectionId: jobManagerConnection
249253
job:
250-
metrics:
251-
connectionId: metricsConnection
252-
retentionInHours: 12
253-
keys:
254-
- key: "public:emp:C:Throughput"
255-
retentionInHours: 4
256-
labels:
257-
schema: public
258-
table: emp
259-
op: C
260-
- key: "public:emp:U:Throughput"
261-
retentionInHours: 4
262-
labels:
263-
schema: public
264-
table: emp
265-
op: U
266-
- key: "public:emp:D:Throughput"
267-
retentionInHours: 4
268-
labels:
269-
schema: public
270-
table: emp
271-
op: D
272-
- key: "public:emp:Latency"
273-
retentionInHours: 4
274-
labels:
275-
schema: public
276-
table: emp
254+
#metrics:
255+
#connectionId: metricsConnection
256+
#retentionInHours: 12
257+
#keys:
258+
#- key: "public:emp:C:Throughput"
259+
#retentionInHours: 4
260+
#labels:
261+
#schema: public
262+
#table: emp
263+
#op: C
264+
#- key: "public:emp:U:Throughput"
265+
#retentionInHours: 4
266+
#labels:
267+
#schema: public
268+
#table: emp
269+
#op: U
270+
#- key: "public:emp:D:Throughput"
271+
#retentionInHours: 4
272+
#labels:
273+
#schema: public
274+
#table: emp
275+
#op: D
276+
#- key: "public:emp:Latency"
277+
#retentionInHours: 4
278+
#labels:
279+
#schema: public
280+
#table: emp
277281
jobConfig:
278-
- name: RedisConnect-postgres
282+
- name: postgres-job
279283
config: JobConfig.yml
280284
variables:
281285
database: RedisConnect
@@ -293,9 +297,9 @@ job:
293297
### Sample JobManager.yml under redis-connect-postgres/config/samples/postgres folder
294298
295299
```yml
296-
connectionId: jobConfigConnection
297-
metricsReporter:
298-
- REDIS_TS_METRICS_REPORTER
300+
connectionId: jobManagerConnection
301+
#metricsReporter:
302+
#- REDIS_TS_METRICS_REPORTER
299303
```
300304

301305
</p>

‎connectors/postgres/demo/config/samples/loader/JobConfig.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ jobId: ${jobId}
22
producerConfig:
33
producerId: RDB_INITIAL_LOAD_PRODUCER
44
connectionId: RDBConnection
5-
batchSize: 5000
5+
batchSize: 500
66
startRecord: ${startRecord}
77
endRecord: ${endRecord}
88
loadQuery: ${loadQuery}
@@ -27,3 +27,5 @@ pipelineConfig:
2727
metricEnabled: false
2828
async: true
2929
checkpoint: "${jobId}"
30+
31+

‎connectors/postgres/demo/config/samples/loader/JobManager.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
connectionId: jobConfigConnection
2-
metricsReporter:
3-
- REDIS_TS_METRICS_REPORTER
1+
connectionId: jobManagerConnection
2+
#metricsReporter:
3+
#- REDIS_TS_METRICS_REPORTER

‎connectors/postgres/demo/config/samples/loader/Setup.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
connectionId: jobConfigConnection
1+
connectionId: jobManagerConnection
22
job:
33
jobConfig:
44
- name: initial_load
55
config: TaskCreator.yml
6+
7+

‎connectors/postgres/demo/config/samples/loader/TaskCreator.yml

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
jobId: ${jobId}
22
jobType: TASK_CREATOR
33
producerConfig:
4-
producerId: RDB_TASK_CREATOR
5-
connectionId: RDBConnection
4+
producerId: RDB_TASK_CREATOR # internal id
5+
connectionId: RDBConnection # source connectionId same as env.yml
66
jobs:
7-
- table: public.emp
8-
numJobs: 5
7+
- table: public.emp # schema.table
8+
numJobs: 1
99
config: JobConfig.yml
1010
loadQuery: "select * from (select Row_Number() over (order by (select 0)) as RowIndex, * from public.emp) as emps Where emps.RowIndex >= ${start} and emps.RowIndex <= ${end}"
11-

‎connectors/postgres/demo/config/samples/loader/env.yml

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,24 @@
11
connections:
2-
- id: jobConfigConnection
2+
- id: jobManagerConnection #Redis Connect Job Metadata connection
33
type: Redis
4-
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
5-
- id: targetConnection
4+
url: redis://127.0.0.1:14001 #this is based on lettuce uri syntax
5+
jobmanager.username: ${REDISCONNECT_JOBMANAGER_USERNAME} #this can be overridden by an env variable or a property file
6+
jobmanager.password: ${REDISCONNECT_JOBMANAGER_PASSWORD} #this can be overridden by an env variable or a property file
7+
#credentials.file.path: <path to <redisconnect_credentials_jobmanager_<job_name> e.g. /var/secrets/jobmanager> when username and password are not provided here
8+
- id: targetConnection #target Redis connection
69
type: Redis
7-
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14000
8-
- id: metricsConnection
9-
type: Redis
10-
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
10+
url: redis://127.0.0.1:14000 #this is based on lettuce uri syntax
11+
target.username: ${REDISCONNECT_TARGET_USERNAME} #this can be overridden by an env variable or a property file
12+
target.password: ${REDISCONNECT_TARGET_PASSWORD} #this can be overridden by an env variable or a property file
13+
#credentials.file.path: <path to <redisconnect_credentials_redis_<job_name> e.g. /var/secrets/redis> when username and password are not provided here
1114
- id: RDBConnection
1215
type: RDB
1316
name: RedisConnect #database pool name
1417
database: RedisConnect #database
15-
url: "jdbc:postgresql://127.0.0.1:5432/RedisConnect"
18+
url: "jdbc:postgresql://127.0.0.1:5432/RedisConnect" #this is jdbc client driver specific, and it can contain any supported parameters
1619
host: 127.0.0.1
1720
port: 5432
18-
username: ${REDISCONNECT_SOURCE_USERNAME}
19-
password: ${REDISCONNECT_SOURCE_PASSWORD}
21+
source.username: ${REDISCONNECT_SOURCE_USERNAME} #this can be overridden by an env variable or a property file
22+
source.password: ${REDISCONNECT_SOURCE_PASSWORD} #this can be overridden by an env variable or a property file
23+
#credentials.file.path: <path to <redisconnect_credentials_postgresql_<job_name> e.g. /var/secrets/postgresql> when username and password are not provided here
24+
#credentials.rotation.eventlistener.enabled: false #uncomment and make it true if you need credentials rotation support

‎connectors/postgres/demo/config/samples/loader/logback.xml

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
<configuration debug="true" scan="true" scanPeriod="15 seconds">
2+
3+
<property name="LOG_REDIS_CONNECT_PATH" value="logs/redis-connect.log"/>
4+
<property name="LOG_REDIS_CONNECT_MANAGER_PATH" value="logs/redis-connect-manager.log"/>
5+
<property name="LOG_REDIS_CONNECT_HEARTBEAT_PATH" value="logs/redis-connect-heartbeat.log"/>
6+
7+
<appender name="REDIS_CONNECT_HEARTBEAT" class="ch.qos.logback.core.rolling.RollingFileAppender">
8+
<file>${LOG_REDIS_CONNECT_HEARTBEAT_PATH}</file>
9+
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
10+
<fileNamePattern>logs/archived/redis-connect-heartbeat.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
11+
<!-- each archived file, size max 10MB -->
12+
<maxFileSize>10MB</maxFileSize>
13+
<!-- total size of all archive files, if total size > 20GB, it will delete old archived file -->
14+
<totalSizeCap>20GB</totalSizeCap>
15+
<!-- 60 days to keep -->
16+
<maxHistory>60</maxHistory>
17+
</rollingPolicy>
18+
<encoder>
19+
<pattern>%d %p %c{1.} [%t] %m%n</pattern>
20+
</encoder>
21+
</appender>
22+
<appender name="REDIS_CONNECT_MANAGER" class="ch.qos.logback.core.rolling.RollingFileAppender">
23+
<file>${LOG_REDIS_CONNECT_MANAGER_PATH}</file>
24+
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
25+
<fileNamePattern>logs/archived/redis-connect-manager.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
26+
<!-- each archived file, size max 10MB -->
27+
<maxFileSize>10MB</maxFileSize>
28+
<!-- total size of all archive files, if total size > 20GB, it will delete old archived file -->
29+
<totalSizeCap>20GB</totalSizeCap>
30+
<!-- 60 days to keep -->
31+
<maxHistory>60</maxHistory>
32+
</rollingPolicy>
33+
<encoder>
34+
<pattern>%d %p %c{1.} [%t] %m%n</pattern>
35+
</encoder>
36+
</appender>
37+
<appender name="REDIS_CONNECT" class="ch.qos.logback.core.rolling.RollingFileAppender">
38+
<file>${LOG_REDIS_CONNECT_PATH}</file>
39+
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
40+
<fileNamePattern>logs/archived/app.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
41+
<!-- each archived file, size max 10MB -->
42+
<maxFileSize>10MB</maxFileSize>
43+
<!-- total size of all archive files, if total size > 20GB, it will delete old archived file -->
44+
<totalSizeCap>20GB</totalSizeCap>
45+
<!-- 60 days to keep -->
46+
<maxHistory>60</maxHistory>
47+
</rollingPolicy>
48+
<encoder>
49+
<pattern>%d %p %c{1.} [%t] %m%n</pattern>
50+
</encoder>
51+
</appender>
52+
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
53+
<encoder>
54+
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
55+
</encoder>
56+
</appender>
57+
58+
<logger name="redis-connect-heartbeat" level="INFO" additivity="false">
59+
<appender-ref ref="REDIS_CONNECT_HEARTBEAT"/>
60+
<appender-ref ref="CONSOLE" />
61+
</logger>
62+
<logger name="redis-connect-manager" level="INFO" additivity="false">
63+
<appender-ref ref="REDIS_CONNECT_MANAGER"/>
64+
<appender-ref ref="CONSOLE" />
65+
</logger>
66+
<logger name="redis-connect" level="INFO" additivity="false">
67+
<appender-ref ref="REDIS_CONNECT"/>
68+
<appender-ref ref="CONSOLE" />
69+
</logger>
70+
<logger name="io.netty" level="OFF" additivity="false">
71+
<appender-ref ref="REDIS_CONNECT"/>
72+
<appender-ref ref="CONSOLE" />
73+
</logger>
74+
<logger name="io.lettuce" level="OFF" additivity="false">
75+
<appender-ref ref="REDIS_CONNECT"/>
76+
<appender-ref ref="CONSOLE" />
77+
</logger>
78+
<logger name="com.zaxxer" level="OFF" additivity="false">
79+
<appender-ref ref="REDIS_CONNECT"/>
80+
<appender-ref ref="CONSOLE"/>
81+
</logger>
82+
<logger name="io.debezium" level="INFO" additivity="false">
83+
<appender-ref ref="REDIS_CONNECT"/>
84+
<appender-ref ref="CONSOLE"/>
85+
</logger>
86+
<logger name="org.apache.kafka" level="OFF" additivity="false">
87+
<appender-ref ref="REDIS_CONNECT"/>
88+
<appender-ref ref="CONSOLE"/>
89+
</logger>
90+
<logger name="org.springframework" level="OFF" additivity="false">
91+
<appender-ref ref="REDIS_CONNECT"/>
92+
<appender-ref ref="CONSOLE"/>
93+
</logger>
94+
95+
<root>
96+
<appender-ref ref="REDIS_CONNECT"/>
97+
<appender-ref ref="REDIS_CONNECT_MANAGER"/>
98+
<appender-ref ref="REDIS_CONNECT_HEARTBEAT"/>
99+
</root>
100+
101+
</configuration>

‎connectors/postgres/demo/config/samples/loader/mappers/mapper1.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,4 @@ tables:
2424
- src: comm
2525
target: Commission
2626
- src: dept
27-
target: Department
27+
target: Department

‎connectors/postgres/demo/config/samples/postgres/JobManager.yml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
11
connectionId: jobConfigConnection
2-
metricsReporter:
3-
- REDIS_TS_METRICS_REPORTER
2+

‎connectors/postgres/demo/config/samples/postgres/Setup.yml

Lines changed: 2 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,7 @@
1-
connectionId: jobConfigConnection
1+
connectionId: jobManagerConnection
22
job:
3-
metrics:
4-
connectionId: metricsConnection
5-
retentionInHours: 12
6-
keys:
7-
- key: "public:emp:C:Throughput"
8-
retentionInHours: 4
9-
labels:
10-
schema: public
11-
table: emp
12-
op: C
13-
- key: "public:emp:U:Throughput"
14-
retentionInHours: 4
15-
labels:
16-
schema: public
17-
table: emp
18-
op: U
19-
- key: "public:emp:D:Throughput"
20-
retentionInHours: 4
21-
labels:
22-
schema: public
23-
table: emp
24-
op: D
25-
- key: "public:emp:Latency"
26-
retentionInHours: 4
27-
labels:
28-
schema: public
29-
table: emp
303
jobConfig:
31-
- name: RedisConnect-postgres
4+
- name: postgres-job
325
config: JobConfig.yml
336
variables:
347
database: RedisConnect

‎connectors/postgres/demo/config/samples/postgres/env.yml

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,25 @@
11
connections:
2-
- id: jobConfigConnection
2+
- id: jobManagerConnection #Redis Connect Job Metadata connection
33
type: Redis
4-
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
5-
- id: targetConnection
4+
url: redis://127.0.0.1:14001 #this is based on lettuce uri syntax
5+
jobmanager.username: ${REDISCONNECT_JOBMANAGER_USERNAME} #this can be overridden by an env variable or a property file
6+
jobmanager.password: ${REDISCONNECT_JOBMANAGER_PASSWORD} #this can be overridden by an env variable or a property file
7+
#credentials.file.path: <path to <redisconnect_credentials_jobmanager_<job_name> e.g. /var/secrets/jobmanager> when username and password are not provided here
8+
- id: targetConnection #target Redis connection
69
type: Redis
7-
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14000
8-
- id: metricsConnection
9-
type: Redis
10-
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
10+
url: redis://127.0.0.1:14000 #this is based on lettuce uri syntax
11+
target.username: ${REDISCONNECT_TARGET_USERNAME} #this can be overridden by an env variable or a property file
12+
target.password: ${REDISCONNECT_TARGET_PASSWORD} #this can be overridden by an env variable or a property file
13+
#credentials.file.path: <path to <redisconnect_credentials_redis_<job_name> e.g. /var/secrets/redis> when username and password are not provided here
1114
- id: RDBConnection
1215
type: RDB
1316
name: RedisConnect #database pool name
1417
database: RedisConnect #database
15-
url: "jdbc:postgresql://127.0.0.1:5432/RedisConnect"
18+
url: "jdbc:postgresql://127.0.0.1:5432/RedisConnect" #this is jdbc client driver specific, and it can contain any supported parameters
1619
host: 127.0.0.1
1720
port: 5432
18-
username: ${REDISCONNECT_SOURCE_USERNAME}
19-
password: ${REDISCONNECT_SOURCE_PASSWORD}
20-
heartbeat.interval.ms: 10000
21-
heartbeat.action.query: "INSERT INTO heartbeat (id, ts) VALUES (1, NOW()) ON CONFLICT(id) DO UPDATE SET ts=EXCLUDED.ts;"
21+
source.username: ${REDISCONNECT_SOURCE_USERNAME} #this can be overridden by an env variable or a property file
22+
source.password: ${REDISCONNECT_SOURCE_PASSWORD} #this can be overridden by an env variable or a property file
23+
#credentials.file.path: <path to <redisconnect_credentials_postgresql_<job_name> e.g. /var/secrets/postgresql> when username and password are not provided here
24+
#heartbeat.interval.ms: 10000 #Workaround for AWS RDS PG WAL space issue
25+
#heartbeat.action.query: "INSERT INTO heartbeat (id, ts) VALUES (1, NOW()) ON CONFLICT(id) DO UPDATE SET ts=EXCLUDED.ts;" #Workaround for AWS RDS PG WAL space issue

0 commit comments

Comments
 (0)