Skip to content

Commit 03e9ceb

Browse files
viragtripathiviragtripathi
committed
Oracle demo/docs initial update.
1 parent d40d83e commit 03e9ceb

29 files changed

+1633
-42
lines changed

‎connectors/oracle/README.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# redis-connect-oracle
2+
3+
redis-connect-oracle is a Redis Connect connector for capturing changes (INSERT, UPDATE and DELETE) from MySQL (source) and writing them to a Redis Enterprise database (Target). The connector uses [Oracle LogMiner](https://docs.oracle.com/cd/B19306_01/server.102/b14215/logminer.htm#i1010243) to read the database redo log.
4+
5+
<p>
6+
The first time redis-connect-oracle connects to a Oracle database, it reads a consistent snapshot of all of the schemas.
7+
When that snapshot is complete, the connector continuously streams the changes that were committed to Oracle and generates a corresponding insert, update or delete event.
8+
All of the events for each tables are recorded in a separate Redis data structure or module of your choice, where they can be easily consumed by applications and services.

‎connectors/oracle/demo/README.md

Lines changed: 582 additions & 0 deletions
Large diffs are not rendered by default.

‎connectors/oracle/demo/config/logback.xml

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
<configuration debug="true" scan="true" scanPeriod="15 seconds">
2+
3+
<property name="START_UP_PATH" value="logs/redis-connect-startup.log"/>
4+
<property name="LOG_PATH" value="logs/redis-connect.log"/>
5+
6+
<appender name="STARTUP" class="ch.qos.logback.core.rolling.RollingFileAppender">
7+
<file>${START_UP_PATH}</file>
8+
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
9+
<fileNamePattern>logs/archived/startup.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
10+
<!-- each archived file, size max 10MB -->
11+
<maxFileSize>10MB</maxFileSize>
12+
<!-- total size of all archive files, if total size > 20GB, it will delete old archived file -->
13+
<totalSizeCap>20GB</totalSizeCap>
14+
<!-- 60 days to keep -->
15+
<maxHistory>60</maxHistory>
16+
</rollingPolicy>
17+
<encoder>
18+
<pattern>%d %p %c{1.} [%t] %m%n</pattern>
19+
</encoder>
20+
</appender>
21+
22+
<appender name="REDISCONNECT" class="ch.qos.logback.core.rolling.RollingFileAppender">
23+
<file>${LOG_PATH}</file>
24+
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
25+
<fileNamePattern>logs/archived/app.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
26+
<!-- each archived file, size max 10MB -->
27+
<maxFileSize>10MB</maxFileSize>
28+
<!-- total size of all archive files, if total size > 20GB, it will delete old archived file -->
29+
<totalSizeCap>20GB</totalSizeCap>
30+
<!-- 60 days to keep -->
31+
<maxHistory>60</maxHistory>
32+
</rollingPolicy>
33+
<encoder>
34+
<pattern>%d %p %c{1.} [%t] %m%n</pattern>
35+
</encoder>
36+
</appender>
37+
38+
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
39+
<encoder>
40+
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
41+
</encoder>
42+
</appender>
43+
44+
<logger name="startup" level="INFO" additivity="false">
45+
<appender-ref ref="STARTUP"/>
46+
<appender-ref ref="CONSOLE" />
47+
</logger>
48+
49+
<logger name="redisconnect" level="INFO" additivity="false">
50+
<appender-ref ref="REDISCONNECT"/>
51+
<appender-ref ref="CONSOLE" />
52+
</logger>
53+
54+
55+
<logger name="com.redislabs" level="INFO" additivity="false">
56+
<appender-ref ref="REDISCONNECT"/>
57+
<appender-ref ref="CONSOLE" />
58+
</logger>
59+
<logger name="io.netty" level="OFF" additivity="false">
60+
<appender-ref ref="REDISCONNECT"/>
61+
<appender-ref ref="CONSOLE" />
62+
</logger>
63+
<logger name="io.lettuce" level="OFF" additivity="false">
64+
<appender-ref ref="REDISCONNECT"/>
65+
<appender-ref ref="CONSOLE" />
66+
</logger>
67+
<logger name="com.zaxxer" level="OFF" additivity="false">
68+
<appender-ref ref="REDISCONNECT"/>
69+
<appender-ref ref="CONSOLE"/>
70+
</logger>
71+
<logger name="io.debezium" level="INFO" additivity="false">
72+
<appender-ref ref="REDISCONNECT"/>
73+
<appender-ref ref="CONSOLE"/>
74+
</logger>
75+
<logger name="org.apache.kafka" level="OFF" additivity="false">
76+
<appender-ref ref="REDISCONNECT"/>
77+
<appender-ref ref="CONSOLE"/>
78+
</logger>
79+
<logger name="org.springframework" level="OFF" additivity="false">
80+
<appender-ref ref="REDISCONNECT"/>
81+
<appender-ref ref="CONSOLE"/>
82+
</logger>
83+
84+
<root>
85+
<appender-ref ref="STARTUP"/>
86+
<appender-ref ref="REDISCONNECT"/>
87+
</root>
88+
89+
</configuration>

‎connectors/oracle/demo/config/samples/loader/FormatterConfig.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
default:
2+
formatter: FREEMARKER_FORMATTER
3+
template: name-value.ftl
4+
dbo.emp:
5+
template: pipe-delimited.ftl

‎connectors/oracle/demo/config/samples/loader/JobConfig.yml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
jobId: ${jobId}
2+
producerConfig:
3+
producerId: RDB_INITIAL_LOAD_PRODUCER
4+
connectionId: RDBConnection
5+
batchSize: 5000
6+
startRecord: ${startRecord}
7+
endRecord: ${endRecord}
8+
loadQuery: ${loadQuery}
9+
table: ${table}
10+
pipelineConfig:
11+
eventTranslator: "RDB_RS_TO_OP_TRANSLATOR"
12+
checkpointConfig:
13+
providerId: REDIS_HASH_CHECKPOINT_READER
14+
connectionId: targetConnection
15+
checkpoint: "${jobId}"
16+
stages:
17+
HashWriteStage:
18+
handlerId: REDIS_HASH_WRITER
19+
connectionId: targetConnection
20+
metricsEnabled: false
21+
prependTableNameToKeys: true
22+
deleteOnKeyUpdate: true
23+
async: true
24+
CheckpointStage:
25+
handlerId: REDIS_HASH_CHECKPOINT_WRITER
26+
connectionId: targetConnection
27+
metricEnabled: false
28+
async: true
29+
checkpoint: "${jobId}"
30+
31+

‎connectors/oracle/demo/config/samples/loader/JobManager.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
connectionId: jobConfigConnection
2+
metricsReporter:
3+
- REDIS_TS_METRICS_REPORTER

‎connectors/oracle/demo/config/samples/loader/Setup.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
connectionId: jobConfigConnection
2+
job:
3+
jobConfig:
4+
- name: initial_load
5+
config: TaskCreator.yml

‎connectors/oracle/demo/config/samples/loader/TaskCreator.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
jobId: ${jobId}
2+
jobType: TASK_CREATOR
3+
producerConfig:
4+
producerId: RDB_TASK_CREATOR # internal id
5+
connectionId: RDBConnection # source connectionId same as env.yml
6+
jobs:
7+
- table: HR.EMPLOYEES # schema.table
8+
numJobs: 5
9+
config: JobConfig.yml
10+
loadQuery: "SELECT * FROM (SELECT e.*, ROW_NUMBER() OVER (ORDER BY EMPLOYEE_ID) RowIndex FROM HR.EMPLOYEES e) WHERE RowIndex BETWEEN ${start} and ${end}"

‎connectors/oracle/demo/config/samples/loader/env.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
connections:
2+
- id: jobConfigConnection
3+
type: Redis
4+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
5+
- id: targetConnection
6+
type: Redis
7+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14000
8+
- id: metricsConnection
9+
type: Redis
10+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
11+
- id: RDBConnection
12+
type: RDB
13+
name: RedisConnect #database pool name
14+
database: ORCLPDB1 #database
15+
url: "jdbc:oracle:thin:@//127.0.0.1:1521/ORCLPDB1" # 12c
16+
#url: "jdbc:oracle:thin:@//127.0.0.1:1522/ORCLPDB1" # 19c
17+
host: 127.0.0.1
18+
port: 1521
19+
username: ${REDISCONNECT_SOURCE_USERNAME}
20+
password: ${REDISCONNECT_SOURCE_PASSWORD}

‎connectors/oracle/demo/config/samples/loader/mappers/mapper1.yml

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
schema: HR
2+
tables:
3+
- table: EMPLOYEES
4+
mapper:
5+
id: employees
6+
processorID: employees
7+
publishBefore: false
8+
columns:
9+
- src: EMPLOYEE_ID
10+
target: EmpId
11+
type: INT
12+
publishBefore: false
13+
- src: FIRST_NAME
14+
target: FName
15+
- src: LAST_NAME
16+
target: LName
17+
- src: EMAIL
18+
target: Email
19+
- src: JOB_ID
20+
target: JobId
21+
- src: SALARY
22+
target: Salary
23+
type: DOUBLE
24+
- table: JOBS
25+
mapper:
26+
id: Jobs
27+
processorID: Jobs
28+
publishBefore: false
29+
passThrough: true
30+
columns:
31+
- src: JOB_ID
32+
target: JobId
33+
type: INT
34+
- table: EMP
35+
mapper:
36+
id: emp
37+
processorID: emp
38+
publishBefore: false
39+
columns:
40+
- src: EMPNO
41+
target: EmployeeNumber
42+
type: INT
43+
publishBefore: false
44+
- src: FNAME
45+
target: FirstName
46+
- src: LNAME
47+
target: LastName
48+
- src: JOB
49+
target: Job
50+
- src: MGR
51+
target: Manager
52+
- src: HIREDATE
53+
target: HireDate
54+
- src: SAL
55+
target: Salary
56+
- src: COMM
57+
target: Commission
58+
- src: DEPT
59+
target: Department

‎connectors/oracle/demo/config/samples/loader/templates/defaultformat.ftl

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<#--
2+
<Operation type="U" table="EMP" schema="SCOTT"><TransactionInfo><RBA>13826</RBA><SeqNum>6</SeqNum><Position>00000000060000013826</Position><TxTime>2013-04-01 23:12:15.000052</TxTime><ReadTime>2013-04-01 16:12:20.276</ReadTime></TransactionInfo><Cols><Col name="empNo">7900</Col><Col name="salary">1000.00</Col></Cols></Operation>-->
3+
<Operation type="${op.getType()}" table="${op.getTable()}" schema="${op.getSchema()}" rba="${op.getRba()}" seq="${op.getSeq()}" pos="${op.getPos()}" txTime="${op.getTxTime()}" readTime="${op.getReadTime()}">
4+
<#list op.getCols().getCol() as col>
5+
<#if col.before ??>
6+
<Col name="${col.getName()}" before="${col.getBefore()}">${col.getValue()}</Col>
7+
<#else>
8+
<Col name="${col.getName()}">${col.getValue()}</Col>
9+
</#if>
10+
</#list>
11+
</Operation>

‎connectors/oracle/demo/config/samples/loader/templates/name-value.ftl

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<#macro compress_single_line><#local captured><#nested></#local>${captured?replace("^\\s+|\\s+$|\\n|\\r", "", "rm")}</#macro>
2+
<@compress_single_line>
3+
<#if delete??>
4+
<#if op.type?matches('D','i')>
5+
<#list op.getCols().getCol()?filter(it -> it??) as col>
6+
${col.name}:
7+
<#if col.before ??>${col.before}</#if>
8+
<#if col?has_next>,</#if>
9+
</#list>
10+
<#else>
11+
<#list op.getCols().getCol()?filter(it -> it??) as col>
12+
${col.name}:
13+
<#if col.changed>
14+
<#if col.before??>${col.before}</#if>
15+
<#else>
16+
<#if col.value??>${col.value}</#if>
17+
</#if>
18+
<#if col?has_next>,</#if>
19+
</#list>
20+
</#if>
21+
<#else>
22+
<#list op.getCols().getCol()?filter(it -> it??) as col>
23+
${col.name}:
24+
<#if col.value ??>${col.value}</#if>
25+
<#if col?has_next>,</#if>
26+
</#list>
27+
</#if>
28+
</@compress_single_line>

‎connectors/oracle/demo/config/samples/loader/templates/pipe-delimited.ftl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
<#list op.getCols().getCol() as col><#if col.value ??>${col.getValue()}|<#else>|</#if></#list>

‎connectors/oracle/demo/config/samples/oracle/FormatterConfig.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
default:
2+
formatter: FREEMARKER_FORMATTER
3+
template: name-value.ftl
4+
dbo.emp:
5+
template: pipe-delimited.ftl

‎connectors/oracle/demo/config/samples/oracle/JobConfig.yml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
jobId: ${jobId}
2+
producerConfig:
3+
producerId: LOG_MINER_EVENT_PRODUCER
4+
connectionId: RDBConnection
5+
metaConnectionId: RDBMetadataConnection
6+
tables:
7+
- HR.EMPLOYEES #schema.table
8+
- HR.JOBS
9+
metricsEnabled: false
10+
pipelineConfig:
11+
eventTranslator: "ORACLE_LOG_MINER_TRANSLATOR"
12+
checkpointConfig:
13+
providerId: REDIS_HASH_CHECKPOINT_READER
14+
connectionId: targetConnection
15+
checkpoint: "${jobId}-${schema}"
16+
stages:
17+
HashWriteStage:
18+
handlerId: REDIS_HASH_WRITER
19+
connectionId: targetConnection
20+
metricsEnabled: false
21+
prependTableNameToKeys: true
22+
deleteOnKeyUpdate: true
23+
async: true
24+
CheckpointStage:
25+
handlerId: REDIS_HASH_CHECKPOINT_WRITER
26+
connectionId: targetConnection
27+
metricEnabled: false
28+
async: true
29+
checkpoint: "${jobId}-${schema}"

‎connectors/oracle/demo/config/samples/oracle/JobManager.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
connectionId: jobConfigConnection
2+
metricsReporter:
3+
- REDIS_TS_METRICS_REPORTER

‎connectors/oracle/demo/config/samples/oracle/Setup.yml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
connectionId: jobConfigConnection
2+
job:
3+
metrics:
4+
connectionId: metricsConnection
5+
retentionInHours: 12
6+
keys:
7+
- key: "HR:EMPLOYEES:C:Throughput"
8+
retentionInHours: 4
9+
labels:
10+
schema: HR
11+
table: EMPLOYEES
12+
op: C
13+
- key: "HR:EMPLOYEES:U:Throughput"
14+
retentionInHours: 4
15+
labels:
16+
schema: HR
17+
table: EMPLOYEES
18+
op: U
19+
- key: "HR:EMPLOYEES:D:Throughput"
20+
retentionInHours: 4
21+
labels:
22+
schema: HR
23+
table: EMPLOYEES
24+
op: D
25+
- key: "HR:EMPLOYEES:Latency"
26+
retentionInHours: 4
27+
labels:
28+
schema: HR
29+
table: EMPLOYEES
30+
jobConfig:
31+
- name: ORACLE-HR
32+
config: JobConfig.yml
33+
variables:
34+
schema: HR

‎connectors/oracle/demo/config/samples/oracle/env.yml

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
connections:
2+
- id: jobConfigConnection
3+
type: Redis
4+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
5+
- id: targetConnection
6+
type: Redis
7+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14000
8+
- id: metricsConnection
9+
type: Redis
10+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
11+
- id: RDBConnection
12+
type: RDB
13+
name: testdb #database name
14+
database: ORCLCDB #database
15+
url: "jdbc:oracle:thin:@//127.0.0.1:1521/ORCLCDB" # 12c
16+
host: 127.0.0.1
17+
port: 1521
18+
username: ${REDISCONNECT_SOURCE1_USERNAME}
19+
password: ${REDISCONNECT_SOURCE1_PASSWORD}
20+
- id: RDBMetadataConnection
21+
type: RDB
22+
name: RedisConnect #database pool name
23+
database: ORCLPDB1 #database
24+
url: "jdbc:oracle:thin:@//127.0.0.1:1521/ORCLPDB1" # 12c
25+
host: 127.0.0.1
26+
port: 1521
27+
username: ${REDISCONNECT_SOURCE2_USERNAME}
28+
password: ${REDISCONNECT_SOURCE2_PASSWORD}

0 commit comments

Comments
 (0)