diff --git a/README.md b/README.md index 6bb289eba8bf9483ccb1ee1c0b6a5c54a7460a69..37a70c0c353953ab2a70e9c919fb33dc1d8d851c 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,194 @@ -Blazer +#Blazer/数据传播者 +Blazer意为**传播者**,主要实现不同数据库之间的数据同步,导出数据库表结构创建语句。 -1、导出数据库表数据到txt文件或者excel +1. 导出数据库表数据到txt/CSV文件(多线程)或者excel(.xlsx/.xls); -2、把txt文件或者excel中的数据导入到数据库表 +2. 把txt文件或者excel中的数据导入到数据库表,支持基于javascript脚本对数据的处理; -3、翻译特定的数据ddl成其他数据库的语法,方便数据迁移 +3. 翻译特定的数据库表结构ddl成其他数据库的语法,方便数据迁移; -4、数据库之间的数据同步 \ No newline at end of file +4. 数据库之间的数据同步; + +5. 目前支持数据有mysql/MariaDB,Greenplum,Oracle,DB2,PostgreSQL; + +------------ + + 项目基于Java Spring Framework框架,简化了系统的配置,引入"管道"(PipeLine)的概念,一个管道包含多个任务,任务可以是数据同步、导入、导出或者表结构的导出。 + + + 在不同的XML中引入不同管道任务,实现不同的功能,其中pipeLine.xml描述管道的功能,需要根据需求进行定义,可以基于以下的命令内容copy出不同脚本功能。 +PipeLineRunner.bat +````bash +@echo off +SET JAVA_HOME=%cd%/jre + +SET JAVA_MARK=PipeLineRunner +SET JAVA_OPTS=" -Xms128m " +SET JAVA_OPTS=%JAVA_OPTS% -Xmx1024m" +rem SET JAVA_OPTS="%JAVA_OPTS% -Dfile.encoding=UTF-8" +SET JAVA_OPTS="%JAVA_OPTS% -Dfile.encoding=GBK" +SET JAVA_OPTS="%JAVA_OPTS% -DjavaMark=%JAVA_MARK%" + +SET JAVA_CONF=./etc +SET JAVA_LIBPATH=./lib +SET JAVA_CLASSPATH=./classes;./bin;%JAVA_CONF% +SET JAVA_MAINCLASS=com.blazer.pipeline.PipeLineRunner +SET JAVA_EXEC=%JAVA_HOME%/bin/java + +rem mk logs dir +if NOT EXIST "./logs" MKDIR "logs" +rem init TEMP_CLASSPATH +SET TEMP_CLASSPATH= +rem new setclasspath.bat +echo SET TEMP_CLASSPATH=%%TEMP_CLASSPATH%%;%%1> setclasspath.bat + +FOR %%i IN (%JAVA_LIBPATH%/*.jar) DO ( +CALL setclasspath.bat %JAVA_LIBPATH%/%%i +) + +SET JAVA_CLASSPATH=%JAVA_CLASSPATH%;%TEMP_CLASSPATH% +rem delete setclasspath.bat +DEL setclasspath.bat +echo %JAVA_CLASSPATH% + +rem Display our environment +echo =============================================================================== +echo Bootstrap Environment +echo. +echo JAVA_CLASSPATH = %JAVA_CLASSPATH% +echo JAVA_CONF = %JAVA_CONF% +echo JAVA_OPTS = %JAVA_OPTS% +echo JAVA_HOME = %JAVA_HOME% +echo JAVA = %JAVA_EXEC% +echo. +%JAVA_EXEC% -version +echo. +echo =============================================================================== +echo. + +%JAVA_EXEC% %JAVA_OPTS% -classpath %JAVA_CLASSPATH% %JAVA_MAINCLASS% --config pipeLine.xml + +echo run finished +PAUSE +```` + +PipeLineRunner.sh +```shell +#!/bin/bash +JAVA_HOME=/opt/java6 + +JAVA_MARK=PipeLineRunner +JAVA_OPTS=" -Xms128m " +JAVA_OPTS="${JAVA_OPTS} -Xmx1024m" +JAVA_OPTS="${JAVA_OPTS} -Dfile.encoding=UTF-8" +JAVA_OPTS="${JAVA_OPTS} -DjavaMark=${JAVA_MARK}" + +JAVA_CONF=./etc +JAVA_LIBPATH=./lib +JAVA_CLASSPATH=./classes:./bin +JAVA_MAINCLASS=com.blazer.pipeline.PipeLineRunner +JAVA_EXEC=${JAVA_HOME}/bin/java + +export JAVA_CLASSPATH +export JAVA_LIBPATH + +for LL in `ls $JAVA_LIBPATH/*.jar` + do + JAVA_CLASSPATH=$JAVA_CLASSPATH:$LL + +done + +export JAVA_CLASSPATH +# Display our environment +echo "=========================================================================" +echo " Bootstrap Environment" +echo "" +echo JAVA_CLASSPATH : ${JAVA_CLASSPATH} +echo JAVA_CONF : ${JAVA_CONF} +echo JAVA_OPTS : ${JAVA_OPTS} +echo JAVA_HOME : ${JAVA_HOME} +echo JAVA : ${JAVA_EXEC} +${JAVA_EXEC} -version +echo "" +echo "=========================================================================" +echo "" + +${JAVA_EXEC} ${JAVA_OPTS} -classpath ${JAVA_CLASSPATH} ${JAVA_MAINCLASS} --config pipeLine.xml + +echo run finished +``` + +实现Oracle到greenplum数据同步 +```xml + + + + + + + + /pipeline/config/applicationConfig.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` diff --git a/blazer-trans/pipeline/transData-Oracle2GreenPlum.xml b/blazer-trans/pipeline/transData-Oracle2GreenPlum.xml index 7f61cc3263b4143cf1b8eb900293b688e9c8bb01..cb522d03950ea264a975914552bb505b09d473ab 100644 --- a/blazer-trans/pipeline/transData-Oracle2GreenPlum.xml +++ b/blazer-trans/pipeline/transData-Oracle2GreenPlum.xml @@ -32,25 +32,19 @@ - + - - + + -