Skip to content

Commit f263b9c

Browse files
viragtripathiviragtripathi
committed
Gemfire config updates
1 parent 705b710 commit f263b9c

File tree

1 file changed

+85
-57
lines changed

1 file changed

+85
-57
lines changed

‎connectors/gemfire/README.md

Lines changed: 85 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,28 @@ Copy the _sample_ directory and it's contents i.e. _yml_ files, _mappers_ and te
108108
#### logging configuration file.
109109
### Sample logback.xml under redis-connect-gemfire/config folder
110110
```xml
111-
<configuration debug="true" scan="true" scanPeriod="30 seconds">
112-
<property name="LOG_PATH" value="logs/cdc-1.log"/>
113-
<appender name="FILE-ROLLING" class="ch.qos.logback.core.rolling.RollingFileAppender">
111+
<configuration debug="true" scan="true" scanPeriod="15 seconds">
112+
113+
<property name="START_UP_PATH" value="logs/redis-connect-startup.log"/>
114+
<property name="LOG_PATH" value="logs/redis-connect.log"/>
115+
116+
<appender name="STARTUP" class="ch.qos.logback.core.rolling.RollingFileAppender">
117+
<file>${START_UP_PATH}</file>
118+
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
119+
<fileNamePattern>logs/archived/startup.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
120+
<!-- each archived file, size max 10MB -->
121+
<maxFileSize>10MB</maxFileSize>
122+
<!-- total size of all archive files, if total size > 20GB, it will delete old archived file -->
123+
<totalSizeCap>20GB</totalSizeCap>
124+
<!-- 60 days to keep -->
125+
<maxHistory>60</maxHistory>
126+
</rollingPolicy>
127+
<encoder>
128+
<pattern>%d %p %c{1.} [%t] %m%n</pattern>
129+
</encoder>
130+
</appender>
131+
132+
<appender name="REDISCONNECT" class="ch.qos.logback.core.rolling.RollingFileAppender">
114133
<file>${LOG_PATH}</file>
115134
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
116135
<fileNamePattern>logs/archived/app.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
@@ -126,18 +145,47 @@ Copy the _sample_ directory and it's contents i.e. _yml_ files, _mappers_ and te
126145
</encoder>
127146
</appender>
128147

129-
<logger name="com.ivoyant" level="INFO" additivity="false">
130-
<appender-ref ref="FILE-ROLLING"/>
148+
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
149+
<encoder>
150+
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
151+
</encoder>
152+
</appender>
153+
154+
<logger name="startup" level="INFO" additivity="false">
155+
<appender-ref ref="STARTUP"/>
156+
<appender-ref ref="CONSOLE" />
131157
</logger>
132-
<logger name="io.netty" level="INFO" additivity="false">
133-
<appender-ref ref="FILE-ROLLING"/>
158+
159+
<logger name="redisconnect" level="INFO" additivity="false">
160+
<appender-ref ref="REDISCONNECT"/>
161+
<appender-ref ref="CONSOLE" />
162+
</logger>
163+
164+
165+
<logger name="com.redislabs" level="INFO" additivity="false">
166+
<appender-ref ref="REDISCONNECT"/>
167+
<appender-ref ref="CONSOLE" />
134168
</logger>
135-
<logger name="io.lettuce" level="INFO" additivity="false">
136-
<appender-ref ref="FILE-ROLLING"/>
169+
<logger name="io.netty" level="OFF" additivity="false">
170+
<appender-ref ref="REDISCONNECT"/>
171+
<appender-ref ref="CONSOLE" />
172+
</logger>
173+
<logger name="io.lettuce" level="OFF" additivity="false">
174+
<appender-ref ref="REDISCONNECT"/>
175+
<appender-ref ref="CONSOLE" />
176+
</logger>
177+
<logger name="org.apache" level="OFF" additivity="false">
178+
<appender-ref ref="REDISCONNECT"/>
179+
<appender-ref ref="CONSOLE"/>
180+
</logger>
181+
<logger name="org.springframework" level="OFF" additivity="false">
182+
<appender-ref ref="REDISCONNECT"/>
183+
<appender-ref ref="CONSOLE"/>
137184
</logger>
138185

139-
<root level="error">
140-
<appender-ref ref="FILE-ROLLING"/>
186+
<root>
187+
<appender-ref ref="STARTUP"/>
188+
<appender-ref ref="REDISCONNECT"/>
141189
</root>
142190

143191
</configuration>
@@ -156,12 +204,15 @@ Redis URI syntax is described [here](https://github.com/lettuce-io/lettuce-core/
156204
### Sample env.yml under redis-connect-gemfire/config/samples/gemfire2redis folder
157205
```yml
158206
connections:
159-
jobConfigConnection:
160-
redisUrl: redis://127.0.0.1:12011
161-
srcConnection:
162-
redisUrl: redis://127.0.0.1:14000
163-
metricsConnection:
164-
redisUrl: redis://127.0.0.1:12011
207+
- id: jobConfigConnection
208+
type: Redis
209+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
210+
- id: targetConnection
211+
type: Redis
212+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14000
213+
- id: metricsConnection
214+
type: Redis
215+
url: redis://${REDISCONNECT_TARGET_USERNAME}:${REDISCONNECT_TARGET_PASSWORD}@127.0.0.1:14001
165216
```
166217
167218
</p>
@@ -175,9 +226,6 @@ connections:
175226
```yml
176227
connectionId: jobConfigConnection
177228
job:
178-
stream: jobStream
179-
configSet: jobConfigs
180-
consumerGroup: jobGroup
181229
metrics:
182230
connectionId: metricsConnection
183231
retentionInHours: 12
@@ -204,7 +252,7 @@ job:
204252
config: JobConfig.yml
205253
variables:
206254
durableClientTimeout: "3000" #This is string value, not a number
207-
gemfireConnectionProvider: com.ivoyant.cdc.connector.gemfire.GemfireConnectionProviderImpl
255+
gemfireConnectionProvider: GemfireConnectionProvider
208256
gemfireConnectionId: gemfireConnection
209257
```
210258
@@ -217,28 +265,9 @@ job:
217265
#### Configuration for Job Reaper and Job Claimer processes.
218266
### Sample JobManager.yml under redis-connect-gemfire/config/samples/gemfire2redis folder
219267
```yml
220-
connectionId: jobConfigConnection # This refers to connectionId from env.yml for Job Config Redis
221-
jobTypeId: jobType1
222-
jobStream: jobStream
223-
jobConfigSet: jobConfigs
224-
initialDelay: 10000
225-
numManagementThreads: 2
226-
metricsReporter:
268+
connectionId: jobConfigConnection
269+
metricsReporter:
227270
- REDIS_TS_METRICS_REPORTER
228-
heartBeatConfig:
229-
key: hb-jobManager
230-
expiry: 30000
231-
jobHeartBeatKeyPrefix: "hb-job:"
232-
jobHeartbeatCheckInterval: 45000
233-
jobClaimerConfig:
234-
initialDelay: 10000
235-
claimInterval: 60000
236-
heartBeatConfig:
237-
key: "hb-job:"
238-
expiry: 30000
239-
maxNumberOfJobs: 2
240-
consumerGroup: jobGroup
241-
batchSize: 1
242271
```
243272
244273
</p>
@@ -251,35 +280,34 @@ jobClaimerConfig:
251280
### Sample JobConfig.yml under redis-connect-gemfire/config/samples/gemfire2redis folder
252281
You can have one or more JobConfig.yml (or with any name e.g. JobConfig-<region_type>.yml) and specify them in the Setup.yml under jobConfig: tag. If specifying more than one table (as below) then make sure maxNumberOfJobs: tag under JobManager.yml is set accordingly e.g. if maxNumberOfJobs: tag is set to 2 then Redis Connect will start 2 cdc jobs under the same JVM instance. If the workload is more and you want to spread out (scale) the cdc jobs then create multiple JobConfig's and specify them in the Setup.yml under jobConfig: tag.
253282
```yml
254-
jobId: ${jobId} #Unique Job Identifier. This value is the job name from Setup.yml
283+
jobId: ${jobId}
255284
producerConfig:
256285
producerId: GEMFIRE_EVENT_PRODUCER
257286
connectionProvider: "${gemfireConnectionProvider}"
258287
connectionId: "${gemfireConnectionId}"
259288
clientId: ${jobId}
260289
clientTimeout: "${durableClientTimeout}" #this has to be quoted, to force the value to be string
261-
metricsKey: "${jobId}:PendingMessageCount"
262290
durable: true
263291
metricsEnabled: false
264292
regions:
265293
- session
266294
pollingInterval: 100
267295
pipelineConfig:
268296
bufferSize: 1024
269-
eventTranslator: ENTRY_EVENT_2_OP_TRANSLATOR
297+
eventTranslator: GEMFIRE_TRANSLATOR
270298
checkpointConfig:
271-
providerId: STRING_CHECKPOINT_READER
272-
connectionId: srcConnection
299+
providerId: GEMFIRE_STRING_CHECKPOINT_READER
300+
connectionId: targetConnection
273301
checkpoint: "${jobId}"
274302
stages:
275-
StringhWriteStage:
276-
handlerId: KV_2_STRING_WRITER
277-
connectionId: srcConnection
303+
StringWriteStage:
304+
handlerId: REDIS_KV_TO_STRING_WRITER
305+
connectionId: targetConnection
278306
metricsEnabled: true
279307
async: true
280308
CheckpointStage:
281-
handlerId: STRING_CP_WRITER
282-
connectionId: srcConnection
309+
handlerId: REDIS_STRING_CHECKPOINT_WRITER
310+
connectionId: targetConnection
283311
metricEnabled: false
284312
async: true
285313
checkpoint: "${jobId}"
@@ -331,14 +359,14 @@ pipelineConfig:
331359
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
332360
xsi:schemaLocation="http://geode.apache.org/schema/cache http://geode.apache.org/schema/cache/cache-1.0.xsd"
333361
version="1.0">
334-
<cache-server port="11111" max-connections="16"/>
335-
362+
<cache-server bind-address="127.0.0.1" port="11111" max-connections="16"/>
363+
336364
<pdx read-serialized="true">
337365
<pdx-serializer>
338366
<class-name>org.apache.geode.pdx.ReflectionBasedAutoSerializer</class-name>
339367
</pdx-serializer>
340368
</pdx>
341-
369+
342370
<region name="checkpoint">
343371
<region-attributes refid="PARTITION">
344372
<key-constraint>java.lang.String</key-constraint>
@@ -377,8 +405,8 @@ pipelineConfig:
377405
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
378406
xsi:schemaLocation="http://geode.apache.org/schema/cache http://geode.apache.org/schema/cache/cache-1.0.xsd"
379407
version="1.0">
380-
<cache-server port="21111" max-connections="16"/>
381-
408+
<cache-server bind-address="127.0.0.1" port="21111" max-connections="16"/>
409+
382410
<region name="checkpoint">
383411
<region-attributes refid="PARTITION">
384412
<key-constraint>java.lang.String</key-constraint>

0 commit comments

Comments
 (0)