[JAVA] Essayez d'utiliser Talend Part 2
Essayez d'utiliser Talend Part 1
A continué
La dernière fois, j'ai tout fait, de l'installation à la création d'un travail simple. Je l'ai fait en référence à l'URL d'un autre site, donc cette fois, je voudrais expliquer en fonction de mon propre écran. J'ai écrit sur la connexion à DB, mais laissez-moi le faire à nouveau.
Créer des métadonnées
Sélectionnez le type de données cible dans les métadonnées de l'onglet Référentiel et créez-le.
Cette fois, j'ai créé un fichier avec un délimiteur à partir d'un fichier CSV séparé par des virgules.
Dans 1, le fichier CSV a été spécifié directement à l'aide du composant tFileInputDelimited, mais il est enregistré en tant que métadonnées pour pouvoir être réutilisé. Il est également possible de se connecter au DB en définissant ici la destination de la connexion DB.
- Omis cette fois
Créer un programme
La dernière fois, nous n'avons utilisé que le composant tMap pour éditer les données, mais cette fois, nous compliquerons un peu le processus.
Jointures et filtres de données
Connectez l'historique des achats et les données client au tMap dans le formulaire ci-dessus.
- Connectez d'abord les principales données de l'historique des achats
Réglez le tMap comme ci-dessus.
- Les éléments de sortie sur le côté extérieur sont votre choix
Vous pouvez rejoindre en faisant glisser et déposer la colonne que vous souhaitez rejoindre. Cette fois, il est combiné avec CustID Par défaut, la jointure doit être une jointure EXTERNE GAUCHE.
Vous pouvez filtrer en cliquant sur le bouton de filtrage en haut de chaque liste de colonnes et en entrant l'expression conditionnelle. Écrivez l'expression conditionnelle de la même manière que Java.
Vous pouvez également utiliser différents composants pour la fonction de filtrage et joindre le traitement en fonction de l'objectif du traitement.
Paramètres variables
Définissez des variables dans l'onglet contextuel.
Cette fois, j'ai créé une variable appelée ctxRegion et l'ai utilisée pour le filtrage.
valeur est la valeur incluse par défaut
Pré-traitement / post-traitement
Utilisez le composant tPerJob pour ajouter un processus de suppression du fichier de sortie avant l'exécution de Job4 (le job créé ci-dessus).
- Le composant tFileOutputDelimited ne provoque pas d'erreur même si le même fichier existe, et il existe un paramètre pour ajouter des données, donc cette fois j'ose désactiver le paramètre ici.
De plus, en tant que méthode de paramétrage de prétraitement, vous pouvez ordonner le traitement en cliquant avec le bouton droit sur la tâche cible → reliant les lignes à partir du déclencheur.
finalement
Vous pouvez voir le code du travail créé en passant à l'onglet de code. Pour ceux qui comprennent Java, il peut être plus facile de comprendre si vous obtenez une erreur. Veuillez vérifier la référence.
package local_project.job4_0_1;
import routines.DataOperation;
import routines.TalendDataGenerator;
import routines.DataQuality;
import routines.Relational;
import routines.DataQualityDependencies;
import routines.Mathematical;
import routines.SQLike;
import routines.Numeric;
import routines.TalendStringUtil;
import routines.TalendString;
import routines.DQTechnical;
import routines.StringHandling;
import routines.DataMasking;
import routines.TalendDate;
import routines.DqStringHandling;
import routines.system.*;
import routines.system.api.*;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.math.BigDecimal;
import java.io.ByteArrayOutputStream;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.ObjectOutputStream;
import java.io.ObjectInputStream;
import java.io.IOException;
import java.util.Comparator;
@SuppressWarnings("unused")
/**
* Job: Job4 Purpose: <br>
* Description: <br>
* @author [email protected]
* @version 7.2.1.20190620_1446
* @status
*/
public class Job4 implements TalendJob {
static {System.setProperty("TalendJob.log", "Job4.log");}
private static org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(Job4.class);
protected static void logIgnoredError(String message, Throwable cause) {
log.error(message, cause);
}
public final Object obj = new Object();
// for transmiting parameters purpose
private Object valueObject = null;
public Object getValueObject() {
return this.valueObject;
}
public void setValueObject(Object valueObject) {
this.valueObject = valueObject;
}
private final static String defaultCharset = java.nio.charset.Charset.defaultCharset().name();
private final static String utf8Charset = "UTF-8";
//contains type for every context property
public class PropertiesWithType extends java.util.Properties {
private static final long serialVersionUID = 1L;
private java.util.Map<String,String> propertyTypes = new java.util.HashMap<>();
public PropertiesWithType(java.util.Properties properties){
super(properties);
}
public PropertiesWithType(){
super();
}
public void setContextType(String key, String type) {
propertyTypes.put(key,type);
}
public String getContextType(String key) {
return propertyTypes.get(key);
}
}
// create and load default properties
private java.util.Properties defaultProps = new java.util.Properties();
// create application properties with default
public class ContextProperties extends PropertiesWithType {
private static final long serialVersionUID = 1L;
public ContextProperties(java.util.Properties properties){
super(properties);
}
public ContextProperties(){
super();
}
public void synchronizeContext(){
if(ctxRegion != null){
this.setProperty("ctxRegion", ctxRegion.toString());
}
}
public String ctxRegion;
public String getCtxRegion(){
return this.ctxRegion;
}
}
protected ContextProperties context = new ContextProperties(); // will be instanciated by MS.
public ContextProperties getContext() {
return this.context;
}
private final String jobVersion = "0.1";
private final String jobName = "Job4";
private final String projectName = "LOCAL_PROJECT";
public Integer errorCode = null;
private String currentComponent = "";
private final java.util.Map<String, Object> globalMap = new java.util.HashMap<String, Object>();
private final static java.util.Map<String, Object> junitGlobalMap = new java.util.HashMap<String, Object>();
private final java.util.Map<String, Long> start_Hash = new java.util.HashMap<String, Long>();
private final java.util.Map<String, Long> end_Hash = new java.util.HashMap<String, Long>();
private final java.util.Map<String, Boolean> ok_Hash = new java.util.HashMap<String, Boolean>();
public final java.util.List<String[]> globalBuffer = new java.util.ArrayList<String[]>();
private RunStat runStat = new RunStat();
// OSGi DataSource
private final static String KEY_DB_DATASOURCES = "KEY_DB_DATASOURCES";
private final static String KEY_DB_DATASOURCES_RAW = "KEY_DB_DATASOURCES_RAW";
public void setDataSources(java.util.Map<String, javax.sql.DataSource> dataSources) {
java.util.Map<String, routines.system.TalendDataSource> talendDataSources = new java.util.HashMap<String, routines.system.TalendDataSource>();
for (java.util.Map.Entry<String, javax.sql.DataSource> dataSourceEntry : dataSources.entrySet()) {
talendDataSources.put(dataSourceEntry.getKey(), new routines.system.TalendDataSource(dataSourceEntry.getValue()));
}
globalMap.put(KEY_DB_DATASOURCES, talendDataSources);
globalMap.put(KEY_DB_DATASOURCES_RAW, new java.util.HashMap<String, javax.sql.DataSource>(dataSources));
}
JobStructureCatcherUtils talendJobLog = new JobStructureCatcherUtils(jobName, "_Sx5zQOmQEemaRIEX-Ts-VA", "0.1");
org.talend.job.audit.JobAuditLogger auditLogger_talendJobLog = null;
private final java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream();
private final java.io.PrintStream errorMessagePS = new java.io.PrintStream(new java.io.BufferedOutputStream(baos));
public String getExceptionStackTrace() {
if ("failure".equals(this.getStatus())) {
errorMessagePS.flush();
return baos.toString();
}
return null;
}
private Exception exception;
public Exception getException() {
if ("failure".equals(this.getStatus())) {
return this.exception;
}
return null;
}
private class TalendException extends Exception {
private static final long serialVersionUID = 1L;
private java.util.Map<String, Object> globalMap = null;
private Exception e = null;
private String currentComponent = null;
private String virtualComponentName = null;
public void setVirtualComponentName (String virtualComponentName){
this.virtualComponentName = virtualComponentName;
}
private TalendException(Exception e, String errorComponent, final java.util.Map<String, Object> globalMap) {
this.currentComponent= errorComponent;
this.globalMap = globalMap;
this.e = e;
}
public Exception getException() {
return this.e;
}
public String getCurrentComponent() {
return this.currentComponent;
}
public String getExceptionCauseMessage(Exception e){
Throwable cause = e;
String message = null;
int i = 10;
while (null != cause && 0 < i--) {
message = cause.getMessage();
if (null == message) {
cause = cause.getCause();
} else {
break;
}
}
if (null == message) {
message = e.getClass().getName();
}
return message;
}
@Override
public void printStackTrace() {
if (!(e instanceof TalendException || e instanceof TDieException)) {
if(virtualComponentName!=null && currentComponent.indexOf(virtualComponentName+"_")==0){
globalMap.put(virtualComponentName+"_ERROR_MESSAGE",getExceptionCauseMessage(e));
}
globalMap.put(currentComponent+"_ERROR_MESSAGE",getExceptionCauseMessage(e));
System.err.println("Exception in component " + currentComponent + " (" + jobName + ")");
}
if (!(e instanceof TDieException)) {
if(e instanceof TalendException){
e.printStackTrace();
} else {
e.printStackTrace();
e.printStackTrace(errorMessagePS);
Job4.this.exception = e;
}
}
if (!(e instanceof TalendException)) {
try {
for (java.lang.reflect.Method m : this.getClass().getEnclosingClass().getMethods()) {
if (m.getName().compareTo(currentComponent + "_error") == 0) {
m.invoke(Job4.this, new Object[] { e , currentComponent, globalMap});
break;
}
}
if(!(e instanceof TDieException)){
}
} catch (Exception e) {
this.e.printStackTrace();
}
}
}
}
public void tFileInputDelimited_2_error(Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException {
end_Hash.put(errorComponent, System.currentTimeMillis());
status = "failure";
tFileInputDelimited_2_onSubJobError(exception, errorComponent, globalMap);
}
public void tMap_1_error(Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException {
end_Hash.put(errorComponent, System.currentTimeMillis());
status = "failure";
tFileInputDelimited_2_onSubJobError(exception, errorComponent, globalMap);
}
public void tFileOutputDelimited_1_error(Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException {
end_Hash.put(errorComponent, System.currentTimeMillis());
status = "failure";
tFileInputDelimited_2_onSubJobError(exception, errorComponent, globalMap);
}
public void tFileInputDelimited_1_error(Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException {
end_Hash.put(errorComponent, System.currentTimeMillis());
status = "failure";
tFileInputDelimited_2_onSubJobError(exception, errorComponent, globalMap);
}
public void tAdvancedHash_row2_error(Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException {
end_Hash.put(errorComponent, System.currentTimeMillis());
status = "failure";
tFileInputDelimited_2_onSubJobError(exception, errorComponent, globalMap);
}
public void talendJobLog_error(Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException {
end_Hash.put(errorComponent, System.currentTimeMillis());
status = "failure";
talendJobLog_onSubJobError(exception, errorComponent, globalMap);
}
public void tFileInputDelimited_2_onSubJobError(Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException {
resumeUtil.addLog("SYSTEM_LOG", "NODE:"+ errorComponent, "", Thread.currentThread().getId()+ "", "FATAL", "", exception.getMessage(), ResumeUtil.getExceptionStackTrace(exception),"");
}
public void talendJobLog_onSubJobError(Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException {
resumeUtil.addLog("SYSTEM_LOG", "NODE:"+ errorComponent, "", Thread.currentThread().getId()+ "", "FATAL", "", exception.getMessage(), ResumeUtil.getExceptionStackTrace(exception),"");
}
public static class out1Struct implements routines.system.IPersistableRow<out1Struct> {
final static byte[] commonByteArrayLock_LOCAL_PROJECT_Job4 = new byte[0];
static byte[] commonByteArray_LOCAL_PROJECT_Job4 = new byte[0];
public Integer CustID;
public Integer getCustID () {
return this.CustID;
}
public java.util.Date LastTransactionDate;
public java.util.Date getLastTransactionDate () {
return this.LastTransactionDate;
}
public Integer TotalPurchase;
public Integer getTotalPurchase () {
return this.TotalPurchase;
}
public String Region;
public String getRegion () {
return this.Region;
}
public String Occupation;
public String getOccupation () {
return this.Occupation;
}
private Integer readInteger(ObjectInputStream dis) throws IOException{
Integer intReturn;
int length = 0;
length = dis.readByte();
if (length == -1) {
intReturn = null;
} else {
intReturn = dis.readInt();
}
return intReturn;
}
private void writeInteger(Integer intNum, ObjectOutputStream dos) throws IOException{
if(intNum == null) {
dos.writeByte(-1);
} else {
dos.writeByte(0);
dos.writeInt(intNum);
}
}
private java.util.Date readDate(ObjectInputStream dis) throws IOException{
java.util.Date dateReturn = null;
int length = 0;
length = dis.readByte();
if (length == -1) {
dateReturn = null;
} else {
dateReturn = new Date(dis.readLong());
}
return dateReturn;
}
private void writeDate(java.util.Date date1, ObjectOutputStream dos) throws IOException{
if(date1 == null) {
dos.writeByte(-1);
} else {
dos.writeByte(0);
dos.writeLong(date1.getTime());
}
}
private String readString(ObjectInputStream dis) throws IOException{
String strReturn = null;
int length = 0;
length = dis.readInt();
if (length == -1) {
strReturn = null;
} else {
if(length > commonByteArray_LOCAL_PROJECT_Job4.length) {
if(length < 1024 && commonByteArray_LOCAL_PROJECT_Job4.length == 0) {
commonByteArray_LOCAL_PROJECT_Job4 = new byte[1024];
} else {
commonByteArray_LOCAL_PROJECT_Job4 = new byte[2 * length];
}
}
dis.readFully(commonByteArray_LOCAL_PROJECT_Job4, 0, length);
strReturn = new String(commonByteArray_LOCAL_PROJECT_Job4, 0, length, utf8Charset);
}
return strReturn;
}
private void writeString(String str, ObjectOutputStream dos) throws IOException{
if(str == null) {
dos.writeInt(-1);
} else {
byte[] byteArray = str.getBytes(utf8Charset);
dos.writeInt(byteArray.length);
dos.write(byteArray);
}
}
public void readData(ObjectInputStream dis) {
synchronized(commonByteArrayLock_LOCAL_PROJECT_Job4) {
try {
int length = 0;
this.CustID = readInteger(dis);
this.LastTransactionDate = readDate(dis);
this.TotalPurchase = readInteger(dis);
this.Region = readString(dis);
this.Occupation = readString(dis);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
public void writeData(ObjectOutputStream dos) {
try {
// Integer
writeInteger(this.CustID,dos);
// java.util.Date
writeDate(this.LastTransactionDate,dos);
// Integer
writeInteger(this.TotalPurchase,dos);
// String
writeString(this.Region,dos);
// String
writeString(this.Occupation,dos);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(super.toString());
sb.append("[");
sb.append("CustID="+String.valueOf(CustID));
sb.append(",LastTransactionDate="+String.valueOf(LastTransactionDate));
sb.append(",TotalPurchase="+String.valueOf(TotalPurchase));
sb.append(",Region="+Region);
sb.append(",Occupation="+Occupation);
sb.append("]");
return sb.toString();
}
public String toLogString(){
StringBuilder sb = new StringBuilder();
if(CustID == null){
sb.append("<null>");
}else{
sb.append(CustID);
}
sb.append("|");
if(LastTransactionDate == null){
sb.append("<null>");
}else{
sb.append(LastTransactionDate);
}
sb.append("|");
if(TotalPurchase == null){
sb.append("<null>");
}else{
sb.append(TotalPurchase);
}
sb.append("|");
if(Region == null){
sb.append("<null>");
}else{
sb.append(Region);
}
sb.append("|");
if(Occupation == null){
sb.append("<null>");
}else{
sb.append(Occupation);
}
sb.append("|");
return sb.toString();
}
/**
* Compare keys
*/
public int compareTo(out1Struct other) {
int returnValue = -1;
return returnValue;
}
private int checkNullsAndCompare(Object object1, Object object2) {
int returnValue = 0;
if (object1 instanceof Comparable && object2 instanceof Comparable) {
returnValue = ((Comparable) object1).compareTo(object2);
} else if (object1 != null && object2 != null) {
returnValue = compareStrings(object1.toString(), object2.toString());
} else if (object1 == null && object2 != null) {
returnValue = 1;
} else if (object1 != null && object2 == null) {
returnValue = -1;
} else {
returnValue = 0;
}
return returnValue;
}
private int compareStrings(String string1, String string2) {
return string1.compareTo(string2);
}
}
public static class row1Struct implements routines.system.IPersistableRow<row1Struct> {
final static byte[] commonByteArrayLock_LOCAL_PROJECT_Job4 = new byte[0];
static byte[] commonByteArray_LOCAL_PROJECT_Job4 = new byte[0];
public Integer CustID;
public Integer getCustID () {
return this.CustID;
}
public java.util.Date LastTransactionDate;
public java.util.Date getLastTransactionDate () {
return this.LastTransactionDate;
}
public Integer TotalPurchase;
public Integer getTotalPurchase () {
return this.TotalPurchase;
}
public Integer PrefID;
public Integer getPrefID () {
return this.PrefID;
}
public String Pref;
public String getPref () {
return this.Pref;
}
private Integer readInteger(ObjectInputStream dis) throws IOException{
Integer intReturn;
int length = 0;
length = dis.readByte();
if (length == -1) {
intReturn = null;
} else {
intReturn = dis.readInt();
}
return intReturn;
}
private void writeInteger(Integer intNum, ObjectOutputStream dos) throws IOException{
if(intNum == null) {
dos.writeByte(-1);
} else {
dos.writeByte(0);
dos.writeInt(intNum);
}
}
private java.util.Date readDate(ObjectInputStream dis) throws IOException{
java.util.Date dateReturn = null;
int length = 0;
length = dis.readByte();
if (length == -1) {
dateReturn = null;
} else {
dateReturn = new Date(dis.readLong());
}
return dateReturn;
}
private void writeDate(java.util.Date date1, ObjectOutputStream dos) throws IOException{
if(date1 == null) {
dos.writeByte(-1);
} else {
dos.writeByte(0);
dos.writeLong(date1.getTime());
}
}
private String readString(ObjectInputStream dis) throws IOException{
String strReturn = null;
int length = 0;
length = dis.readInt();
if (length == -1) {
strReturn = null;
} else {
if(length > commonByteArray_LOCAL_PROJECT_Job4.length) {
if(length < 1024 && commonByteArray_LOCAL_PROJECT_Job4.length == 0) {
commonByteArray_LOCAL_PROJECT_Job4 = new byte[1024];
} else {
commonByteArray_LOCAL_PROJECT_Job4 = new byte[2 * length];
}
}
dis.readFully(commonByteArray_LOCAL_PROJECT_Job4, 0, length);
strReturn = new String(commonByteArray_LOCAL_PROJECT_Job4, 0, length, utf8Charset);
}
return strReturn;
}
private void writeString(String str, ObjectOutputStream dos) throws IOException{
if(str == null) {
dos.writeInt(-1);
} else {
byte[] byteArray = str.getBytes(utf8Charset);
dos.writeInt(byteArray.length);
dos.write(byteArray);
}
}
public void readData(ObjectInputStream dis) {
synchronized(commonByteArrayLock_LOCAL_PROJECT_Job4) {
try {
int length = 0;
this.CustID = readInteger(dis);
this.LastTransactionDate = readDate(dis);
this.TotalPurchase = readInteger(dis);
this.PrefID = readInteger(dis);
this.Pref = readString(dis);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
public void writeData(ObjectOutputStream dos) {
try {
// Integer
writeInteger(this.CustID,dos);
// java.util.Date
writeDate(this.LastTransactionDate,dos);
// Integer
writeInteger(this.TotalPurchase,dos);
// Integer
writeInteger(this.PrefID,dos);
// String
writeString(this.Pref,dos);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(super.toString());
sb.append("[");
sb.append("CustID="+String.valueOf(CustID));
sb.append(",LastTransactionDate="+String.valueOf(LastTransactionDate));
sb.append(",TotalPurchase="+String.valueOf(TotalPurchase));
sb.append(",PrefID="+String.valueOf(PrefID));
sb.append(",Pref="+Pref);
sb.append("]");
return sb.toString();
}
public String toLogString(){
StringBuilder sb = new StringBuilder();
if(CustID == null){
sb.append("<null>");
}else{
sb.append(CustID);
}
sb.append("|");
if(LastTransactionDate == null){
sb.append("<null>");
}else{
sb.append(LastTransactionDate);
}
sb.append("|");
if(TotalPurchase == null){
sb.append("<null>");
}else{
sb.append(TotalPurchase);
}
sb.append("|");
if(PrefID == null){
sb.append("<null>");
}else{
sb.append(PrefID);
}
sb.append("|");
if(Pref == null){
sb.append("<null>");
}else{
sb.append(Pref);
}
sb.append("|");
return sb.toString();
}
/**
* Compare keys
*/
public int compareTo(row1Struct other) {
int returnValue = -1;
return returnValue;
}
private int checkNullsAndCompare(Object object1, Object object2) {
int returnValue = 0;
if (object1 instanceof Comparable && object2 instanceof Comparable) {
returnValue = ((Comparable) object1).compareTo(object2);
} else if (object1 != null && object2 != null) {
returnValue = compareStrings(object1.toString(), object2.toString());
} else if (object1 == null && object2 != null) {
returnValue = 1;
} else if (object1 != null && object2 == null) {
returnValue = -1;
} else {
returnValue = 0;
}
return returnValue;
}
private int compareStrings(String string1, String string2) {
return string1.compareTo(string2);
}
}
public static class after_tFileInputDelimited_2Struct implements routines.system.IPersistableRow<after_tFileInputDelimited_2Struct> {
final static byte[] commonByteArrayLock_LOCAL_PROJECT_Job4 = new byte[0];
static byte[] commonByteArray_LOCAL_PROJECT_Job4 = new byte[0];
public Integer CustID;
public Integer getCustID () {
return this.CustID;
}
public java.util.Date LastTransactionDate;
public java.util.Date getLastTransactionDate () {
return this.LastTransactionDate;
}
public Integer TotalPurchase;
public Integer getTotalPurchase () {
return this.TotalPurchase;
}
public Integer PrefID;
public Integer getPrefID () {
return this.PrefID;
}
public String Pref;
public String getPref () {
return this.Pref;
}
private Integer readInteger(ObjectInputStream dis) throws IOException{
Integer intReturn;
int length = 0;
length = dis.readByte();
if (length == -1) {
intReturn = null;
} else {
intReturn = dis.readInt();
}
return intReturn;
}
private void writeInteger(Integer intNum, ObjectOutputStream dos) throws IOException{
if(intNum == null) {
dos.writeByte(-1);
} else {
dos.writeByte(0);
dos.writeInt(intNum);
}
}
private java.util.Date readDate(ObjectInputStream dis) throws IOException{
java.util.Date dateReturn = null;
int length = 0;
length = dis.readByte();
if (length == -1) {
dateReturn = null;
} else {
dateReturn = new Date(dis.readLong());
}
return dateReturn;
}
private void writeDate(java.util.Date date1, ObjectOutputStream dos) throws IOException{
if(date1 == null) {
dos.writeByte(-1);
} else {
dos.writeByte(0);
dos.writeLong(date1.getTime());
}
}
private String readString(ObjectInputStream dis) throws IOException{
String strReturn = null;
int length = 0;
length = dis.readInt();
if (length == -1) {
strReturn = null;
} else {
if(length > commonByteArray_LOCAL_PROJECT_Job4.length) {
if(length < 1024 && commonByteArray_LOCAL_PROJECT_Job4.length == 0) {
commonByteArray_LOCAL_PROJECT_Job4 = new byte[1024];
} else {
commonByteArray_LOCAL_PROJECT_Job4 = new byte[2 * length];
}
}
dis.readFully(commonByteArray_LOCAL_PROJECT_Job4, 0, length);
strReturn = new String(commonByteArray_LOCAL_PROJECT_Job4, 0, length, utf8Charset);
}
return strReturn;
}
private void writeString(String str, ObjectOutputStream dos) throws IOException{
if(str == null) {
dos.writeInt(-1);
} else {
byte[] byteArray = str.getBytes(utf8Charset);
dos.writeInt(byteArray.length);
dos.write(byteArray);
}
}
public void readData(ObjectInputStream dis) {
synchronized(commonByteArrayLock_LOCAL_PROJECT_Job4) {
try {
int length = 0;
this.CustID = readInteger(dis);
this.LastTransactionDate = readDate(dis);
this.TotalPurchase = readInteger(dis);
this.PrefID = readInteger(dis);
this.Pref = readString(dis);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
public void writeData(ObjectOutputStream dos) {
try {
// Integer
writeInteger(this.CustID,dos);
// java.util.Date
writeDate(this.LastTransactionDate,dos);
// Integer
writeInteger(this.TotalPurchase,dos);
// Integer
writeInteger(this.PrefID,dos);
// String
writeString(this.Pref,dos);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(super.toString());
sb.append("[");
sb.append("CustID="+String.valueOf(CustID));
sb.append(",LastTransactionDate="+String.valueOf(LastTransactionDate));
sb.append(",TotalPurchase="+String.valueOf(TotalPurchase));
sb.append(",PrefID="+String.valueOf(PrefID));
sb.append(",Pref="+Pref);
sb.append("]");
return sb.toString();
}
public String toLogString(){
StringBuilder sb = new StringBuilder();
if(CustID == null){
sb.append("<null>");
}else{
sb.append(CustID);
}
sb.append("|");
if(LastTransactionDate == null){
sb.append("<null>");
}else{
sb.append(LastTransactionDate);
}
sb.append("|");
if(TotalPurchase == null){
sb.append("<null>");
}else{
sb.append(TotalPurchase);
}
sb.append("|");
if(PrefID == null){
sb.append("<null>");
}else{
sb.append(PrefID);
}
sb.append("|");
if(Pref == null){
sb.append("<null>");
}else{
sb.append(Pref);
}
sb.append("|");
return sb.toString();
}
/**
* Compare keys
*/
public int compareTo(after_tFileInputDelimited_2Struct other) {
int returnValue = -1;
return returnValue;
}
private int checkNullsAndCompare(Object object1, Object object2) {
int returnValue = 0;
if (object1 instanceof Comparable && object2 instanceof Comparable) {
returnValue = ((Comparable) object1).compareTo(object2);
} else if (object1 != null && object2 != null) {
returnValue = compareStrings(object1.toString(), object2.toString());
} else if (object1 == null && object2 != null) {
returnValue = 1;
} else if (object1 != null && object2 == null) {
returnValue = -1;
} else {
returnValue = 0;
}
return returnValue;
}
private int compareStrings(String string1, String string2) {
return string1.compareTo(string2);
}
}
public void tFileInputDelimited_2Process(final java.util.Map<String, Object> globalMap) throws TalendException {
globalMap.put("tFileInputDelimited_2_SUBPROCESS_STATE", 0);
final boolean execStat = this.execStat;
String iterateId = "";
String currentComponent = "";
java.util.Map<String, Object> resourceMap = new java.util.HashMap<String, Object>();
try {
// TDI-39566 avoid throwing an useless Exception
boolean resumeIt = true;
if (globalResumeTicket == false && resumeEntryMethodName != null) {
String currentMethodName = new java.lang.Exception().getStackTrace()[0].getMethodName();
resumeIt = resumeEntryMethodName.equals(currentMethodName);
}
if (resumeIt || globalResumeTicket) { //start the resume
globalResumeTicket = true;
tFileInputDelimited_1Process(globalMap);
row1Struct row1 = new row1Struct();
out1Struct out1 = new out1Struct();
/**
* [tFileOutputDelimited_1 begin ] start
*/
ok_Hash.put("tFileOutputDelimited_1", false);
start_Hash.put("tFileOutputDelimited_1", System.currentTimeMillis());
currentComponent="tFileOutputDelimited_1";
if (enableLogStash) {
if(resourceMap.get("inIterateVComp") == null){
if(enableLogStash) {
runStat.logStatOnConnection("out1"+iterateId, 0, 0);
}
}
}
int tos_count_tFileOutputDelimited_1 = 0;
if(log.isDebugEnabled())
log.debug("tFileOutputDelimited_1 - " + ("Start to work.") );
if (log.isDebugEnabled()) {
class BytesLimit65535_tFileOutputDelimited_1{
public void limitLog4jByte() throws Exception{
StringBuilder log4jParamters_tFileOutputDelimited_1 = new StringBuilder();
log4jParamters_tFileOutputDelimited_1.append("Parameters:");
log4jParamters_tFileOutputDelimited_1.append("USESTREAM" + " = " + "false");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("FILENAME" + " = " + "\"C:/Users/hryk.hoshi/Downloads/out_\"+context.ctxRegion+\".csv\"");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("ROWSEPARATOR" + " = " + "\"\\n\"");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("FIELDSEPARATOR" + " = " + "\";\"");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("APPEND" + " = " + "false");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("INCLUDEHEADER" + " = " + "true");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("COMPRESS" + " = " + "false");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("ADVANCED_SEPARATOR" + " = " + "false");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("CSV_OPTION" + " = " + "false");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("CREATE" + " = " + "true");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("SPLIT" + " = " + "false");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("FLUSHONROW" + " = " + "false");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("ROW_MODE" + " = " + "false");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("ENCODING" + " = " + "\"UTF-8\"");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("DELETE_EMPTYFILE" + " = " + "false");
log4jParamters_tFileOutputDelimited_1.append(" | ");
log4jParamters_tFileOutputDelimited_1.append("FILE_EXIST_EXCEPTION" + " = " + "true");
log4jParamters_tFileOutputDelimited_1.append(" | ");
if(log.isDebugEnabled())
log.debug("tFileOutputDelimited_1 - " + (log4jParamters_tFileOutputDelimited_1) );
}
}
new BytesLimit65535_tFileOutputDelimited_1().limitLog4jByte();
}
if(enableLogStash) {
talendJobLog.addComponentMessage("tFileOutputDelimited_1", "tFileOutputDelimited");
talendJobLogProcess(globalMap);
}
String fileName_tFileOutputDelimited_1 = "";
fileName_tFileOutputDelimited_1 = (new java.io.File("C:/Users/hryk.hoshi/Downloads/out_"+context.ctxRegion+".csv")).getAbsolutePath().replace("\\","/");
String fullName_tFileOutputDelimited_1 = null;
String extension_tFileOutputDelimited_1 = null;
String directory_tFileOutputDelimited_1 = null;
if((fileName_tFileOutputDelimited_1.indexOf("/") != -1)) {
if(fileName_tFileOutputDelimited_1.lastIndexOf(".") < fileName_tFileOutputDelimited_1.lastIndexOf("/")) {
fullName_tFileOutputDelimited_1 = fileName_tFileOutputDelimited_1;
extension_tFileOutputDelimited_1 = "";
} else {
fullName_tFileOutputDelimited_1 = fileName_tFileOutputDelimited_1.substring(0, fileName_tFileOutputDelimited_1.lastIndexOf("."));
extension_tFileOutputDelimited_1 = fileName_tFileOutputDelimited_1.substring(fileName_tFileOutputDelimited_1.lastIndexOf("."));
}
directory_tFileOutputDelimited_1 = fileName_tFileOutputDelimited_1.substring(0, fileName_tFileOutputDelimited_1.lastIndexOf("/"));
} else {
if(fileName_tFileOutputDelimited_1.lastIndexOf(".") != -1) {
fullName_tFileOutputDelimited_1 = fileName_tFileOutputDelimited_1.substring(0, fileName_tFileOutputDelimited_1.lastIndexOf("."));
extension_tFileOutputDelimited_1 = fileName_tFileOutputDelimited_1.substring(fileName_tFileOutputDelimited_1.lastIndexOf("."));
} else {
fullName_tFileOutputDelimited_1 = fileName_tFileOutputDelimited_1;
extension_tFileOutputDelimited_1 = "";
}
directory_tFileOutputDelimited_1 = "";
}
boolean isFileGenerated_tFileOutputDelimited_1 = true;
java.io.File filetFileOutputDelimited_1 = new java.io.File(fileName_tFileOutputDelimited_1);
globalMap.put("tFileOutputDelimited_1_FILE_NAME",fileName_tFileOutputDelimited_1);
if(filetFileOutputDelimited_1.exists()){
throw new RuntimeException("The particular file \""+filetFileOutputDelimited_1.getAbsoluteFile() +
"\" already exist. If you want to overwrite the file, please uncheck the" +
" \"Throw an error if the file already exist\" option in Advanced settings.");
}
int nb_line_tFileOutputDelimited_1 = 0;
int splitedFileNo_tFileOutputDelimited_1 = 0;
int currentRow_tFileOutputDelimited_1 = 0;
final String OUT_DELIM_tFileOutputDelimited_1 = /** Start field tFileOutputDelimited_1:FIELDSEPARATOR */";"/** End field tFileOutputDelimited_1:FIELDSEPARATOR */;
final String OUT_DELIM_ROWSEP_tFileOutputDelimited_1 = /** Start field tFileOutputDelimited_1:ROWSEPARATOR */"\n"/** End field tFileOutputDelimited_1:ROWSEPARATOR */;
//create directory only if not exists
if(directory_tFileOutputDelimited_1 != null && directory_tFileOutputDelimited_1.trim().length() != 0) {
java.io.File dir_tFileOutputDelimited_1 = new java.io.File(directory_tFileOutputDelimited_1);
if(!dir_tFileOutputDelimited_1.exists()) {
log.info("tFileOutputDelimited_1 - Creating directory '" + dir_tFileOutputDelimited_1.getCanonicalPath() +"'.");
dir_tFileOutputDelimited_1.mkdirs();
log.info("tFileOutputDelimited_1 - The directory '"+ dir_tFileOutputDelimited_1.getCanonicalPath() + "' has been created successfully.");
}
}
//routines.system.Row
java.io.Writer outtFileOutputDelimited_1 = null;
java.io.File fileToDelete_tFileOutputDelimited_1 = new java.io.File(fileName_tFileOutputDelimited_1);
if(fileToDelete_tFileOutputDelimited_1.exists()) {
fileToDelete_tFileOutputDelimited_1.delete();
}
outtFileOutputDelimited_1 = new java.io.BufferedWriter(new java.io.OutputStreamWriter(
new java.io.FileOutputStream(fileName_tFileOutputDelimited_1, false),"UTF-8"));
if(filetFileOutputDelimited_1.length()==0){
outtFileOutputDelimited_1.write("CustID");
outtFileOutputDelimited_1.write(OUT_DELIM_tFileOutputDelimited_1);
outtFileOutputDelimited_1.write("LastTransactionDate");
outtFileOutputDelimited_1.write(OUT_DELIM_tFileOutputDelimited_1);
outtFileOutputDelimited_1.write("TotalPurchase");
outtFileOutputDelimited_1.write(OUT_DELIM_tFileOutputDelimited_1);
outtFileOutputDelimited_1.write("Region");
outtFileOutputDelimited_1.write(OUT_DELIM_tFileOutputDelimited_1);
outtFileOutputDelimited_1.write("Occupation");
outtFileOutputDelimited_1.write(OUT_DELIM_ROWSEP_tFileOutputDelimited_1);
outtFileOutputDelimited_1.flush();
}
resourceMap.put("out_tFileOutputDelimited_1", outtFileOutputDelimited_1);
resourceMap.put("nb_line_tFileOutputDelimited_1", nb_line_tFileOutputDelimited_1);
/**
* [tFileOutputDelimited_1 begin ] stop
*/
/**
* [tMap_1 begin ] start
*/
ok_Hash.put("tMap_1", false);
start_Hash.put("tMap_1", System.currentTimeMillis());
currentComponent="tMap_1";
if (enableLogStash) {
if(resourceMap.get("inIterateVComp") == null){
if(enableLogStash) {
runStat.logStatOnConnection("row1"+iterateId, 0, 0);
}
}
}
int tos_count_tMap_1 = 0;
if(log.isDebugEnabled())
log.debug("tMap_1 - " + ("Start to work.") );
if (log.isDebugEnabled()) {
class BytesLimit65535_tMap_1{
public void limitLog4jByte() throws Exception{
StringBuilder log4jParamters_tMap_1 = new StringBuilder();
log4jParamters_tMap_1.append("Parameters:");
log4jParamters_tMap_1.append("LINK_STYLE" + " = " + "AUTO");
log4jParamters_tMap_1.append(" | ");
log4jParamters_tMap_1.append("TEMPORARY_DATA_DIRECTORY" + " = " + "");
log4jParamters_tMap_1.append(" | ");
log4jParamters_tMap_1.append("ROWS_BUFFER_SIZE" + " = " + "2000000");
log4jParamters_tMap_1.append(" | ");
log4jParamters_tMap_1.append("CHANGE_HASH_AND_EQUALS_FOR_BIGDECIMAL" + " = " + "true");
log4jParamters_tMap_1.append(" | ");
if(log.isDebugEnabled())
log.debug("tMap_1 - " + (log4jParamters_tMap_1) );
}
}
new BytesLimit65535_tMap_1().limitLog4jByte();
}
if(enableLogStash) {
talendJobLog.addComponentMessage("tMap_1", "tMap");
talendJobLogProcess(globalMap);
}
// ###############################
// # Lookup's keys initialization
int count_row1_tMap_1 = 0;
int count_row2_tMap_1 = 0;
org.talend.designer.components.lookup.memory.AdvancedMemoryLookup<row2Struct> tHash_Lookup_row2 = (org.talend.designer.components.lookup.memory.AdvancedMemoryLookup<row2Struct>)
((org.talend.designer.components.lookup.memory.AdvancedMemoryLookup<row2Struct>)
globalMap.get( "tHash_Lookup_row2" ))
;
row2Struct row2HashKey = new row2Struct();
row2Struct row2Default = new row2Struct();
// ###############################
// ###############################
// # Vars initialization
class Var__tMap_1__Struct {
}
Var__tMap_1__Struct Var__tMap_1 = new Var__tMap_1__Struct();
// ###############################
// ###############################
// # Outputs initialization
int count_out1_tMap_1 = 0;
out1Struct out1_tmp = new out1Struct();
// ###############################
/**
* [tMap_1 begin ] stop
*/
/**
* [tFileInputDelimited_2 begin ] start
*/
ok_Hash.put("tFileInputDelimited_2", false);
start_Hash.put("tFileInputDelimited_2", System.currentTimeMillis());
currentComponent="tFileInputDelimited_2";
int tos_count_tFileInputDelimited_2 = 0;
if(log.isDebugEnabled())
log.debug("tFileInputDelimited_2 - " + ("Start to work.") );
if (log.isDebugEnabled()) {
class BytesLimit65535_tFileInputDelimited_2{
public void limitLog4jByte() throws Exception{
StringBuilder log4jParamters_tFileInputDelimited_2 = new StringBuilder();
log4jParamters_tFileInputDelimited_2.append("Parameters:");
log4jParamters_tFileInputDelimited_2.append("FILENAME" + " = " + "\"C:/Users/hryk.hoshi/Desktop/Talend/handson/Transaction_UTF8_2018Q1.csv\"");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("CSV_OPTION" + " = " + "false");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("ROWSEPARATOR" + " = " + "\"\\n\"");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("FIELDSEPARATOR" + " = " + "\",\"");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("HEADER" + " = " + "1");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("FOOTER" + " = " + "0");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("LIMIT" + " = " + "");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("REMOVE_EMPTY_ROW" + " = " + "false");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("UNCOMPRESS" + " = " + "false");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("DIE_ON_ERROR" + " = " + "false");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("ADVANCED_SEPARATOR" + " = " + "false");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("RANDOM" + " = " + "false");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("TRIMALL" + " = " + "false");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("TRIMSELECT" + " = " + "[{TRIM="+("false")+", SCHEMA_COLUMN="+("CustID")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("LastTransactionDate")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("TotalPurchase")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("PrefID")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Pref")+"}]");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("CHECK_FIELDS_NUM" + " = " + "false");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("CHECK_DATE" + " = " + "false");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("ENCODING" + " = " + "\"UTF-8\"");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("SPLITRECORD" + " = " + "false");
log4jParamters_tFileInputDelimited_2.append(" | ");
log4jParamters_tFileInputDelimited_2.append("ENABLE_DECODE" + " = " + "false");
log4jParamters_tFileInputDelimited_2.append(" | ");
if(log.isDebugEnabled())
log.debug("tFileInputDelimited_2 - " + (log4jParamters_tFileInputDelimited_2) );
}
}
new BytesLimit65535_tFileInputDelimited_2().limitLog4jByte();
}
if(enableLogStash) {
talendJobLog.addComponentMessage("tFileInputDelimited_2", "tFileInputDelimited");
talendJobLogProcess(globalMap);
}
final routines.system.RowState rowstate_tFileInputDelimited_2 = new routines.system.RowState();
int nb_line_tFileInputDelimited_2 = 0;
org.talend.fileprocess.FileInputDelimited fid_tFileInputDelimited_2 = null;
int limit_tFileInputDelimited_2 = -1;
try{
Object filename_tFileInputDelimited_2 = "C:/Users/hryk.hoshi/Desktop/Talend/handson/Transaction_UTF8_2018Q1.csv";
if(filename_tFileInputDelimited_2 instanceof java.io.InputStream){
int footer_value_tFileInputDelimited_2 = 0, random_value_tFileInputDelimited_2 = -1;
if(footer_value_tFileInputDelimited_2 >0 || random_value_tFileInputDelimited_2 > 0){
throw new java.lang.Exception("When the input source is a stream,footer and random shouldn't be bigger than 0.");
}
}
try {
fid_tFileInputDelimited_2 = new org.talend.fileprocess.FileInputDelimited("C:/Users/hryk.hoshi/Desktop/Talend/handson/Transaction_UTF8_2018Q1.csv", "UTF-8",",","\n",false,1,0,
limit_tFileInputDelimited_2
,-1, false);
} catch(java.lang.Exception e) {
log.error("tFileInputDelimited_2 - " +e.getMessage());
System.err.println(e.getMessage());
}
log.info("tFileInputDelimited_2 - Retrieving records from the datasource.");
while (fid_tFileInputDelimited_2!=null && fid_tFileInputDelimited_2.nextRecord()) {
rowstate_tFileInputDelimited_2.reset();
row1 = null;
boolean whetherReject_tFileInputDelimited_2 = false;
row1 = new row1Struct();
try {
int columnIndexWithD_tFileInputDelimited_2 = 0;
String temp = "";
columnIndexWithD_tFileInputDelimited_2 = 0;
temp = fid_tFileInputDelimited_2.get(columnIndexWithD_tFileInputDelimited_2);
if(temp.length() > 0) {
try {
row1.CustID = ParserUtils.parseTo_Integer(temp);
} catch(java.lang.Exception ex_tFileInputDelimited_2) {
rowstate_tFileInputDelimited_2.setException(new RuntimeException(String.format("Couldn't parse value for column '%s' in '%s', value is '%s'. Details: %s",
"CustID", "row1", temp, ex_tFileInputDelimited_2), ex_tFileInputDelimited_2));
}
} else {
row1.CustID = null;
}
columnIndexWithD_tFileInputDelimited_2 = 1;
temp = fid_tFileInputDelimited_2.get(columnIndexWithD_tFileInputDelimited_2);
if(temp.length() > 0) {
try {
row1.LastTransactionDate = ParserUtils.parseTo_Date(temp, "yyyy/MM/dd");
} catch(java.lang.Exception ex_tFileInputDelimited_2) {
rowstate_tFileInputDelimited_2.setException(new RuntimeException(String.format("Couldn't parse value for column '%s' in '%s', value is '%s'. Details: %s",
"LastTransactionDate", "row1", temp, ex_tFileInputDelimited_2), ex_tFileInputDelimited_2));
}
} else {
row1.LastTransactionDate = null;
}
columnIndexWithD_tFileInputDelimited_2 = 2;
temp = fid_tFileInputDelimited_2.get(columnIndexWithD_tFileInputDelimited_2);
if(temp.length() > 0) {
try {
row1.TotalPurchase = ParserUtils.parseTo_Integer(temp);
} catch(java.lang.Exception ex_tFileInputDelimited_2) {
rowstate_tFileInputDelimited_2.setException(new RuntimeException(String.format("Couldn't parse value for column '%s' in '%s', value is '%s'. Details: %s",
"TotalPurchase", "row1", temp, ex_tFileInputDelimited_2), ex_tFileInputDelimited_2));
}
} else {
row1.TotalPurchase = null;
}
columnIndexWithD_tFileInputDelimited_2 = 3;
temp = fid_tFileInputDelimited_2.get(columnIndexWithD_tFileInputDelimited_2);
if(temp.length() > 0) {
try {
row1.PrefID = ParserUtils.parseTo_Integer(temp);
} catch(java.lang.Exception ex_tFileInputDelimited_2) {
rowstate_tFileInputDelimited_2.setException(new RuntimeException(String.format("Couldn't parse value for column '%s' in '%s', value is '%s'. Details: %s",
"PrefID", "row1", temp, ex_tFileInputDelimited_2), ex_tFileInputDelimited_2));
}
} else {
row1.PrefID = null;
}
columnIndexWithD_tFileInputDelimited_2 = 4;
row1.Pref = fid_tFileInputDelimited_2.get(columnIndexWithD_tFileInputDelimited_2);
if(rowstate_tFileInputDelimited_2.getException()!=null) {
throw rowstate_tFileInputDelimited_2.getException();
}
} catch (java.lang.Exception e) {
whetherReject_tFileInputDelimited_2 = true;
log.error("tFileInputDelimited_2 - " +e.getMessage());
System.err.println(e.getMessage());
row1 = null;
}
log.debug("tFileInputDelimited_2 - Retrieving the record " + fid_tFileInputDelimited_2.getRowNumber() + ".");
/**
* [tFileInputDelimited_2 begin ] stop
*/
/**
* [tFileInputDelimited_2 main ] start
*/
currentComponent="tFileInputDelimited_2";
tos_count_tFileInputDelimited_2++;
/**
* [tFileInputDelimited_2 main ] stop
*/
/**
* [tFileInputDelimited_2 process_data_begin ] start
*/
currentComponent="tFileInputDelimited_2";
/**
* [tFileInputDelimited_2 process_data_begin ] stop
*/
// Start of branch "row1"
if(row1 != null) {
/**
* [tMap_1 main ] start
*/
currentComponent="tMap_1";
//row1
//row1
if(enableLogStash) {
runStat.logStatOnConnection("row1"+iterateId,1, 1);
}
if(log.isTraceEnabled()){
log.trace("row1 - " + (row1==null? "": row1.toLogString()));
}
boolean hasCasePrimitiveKeyWithNull_tMap_1 = false;
// ###############################
// # Input tables (lookups)
boolean rejectedInnerJoin_tMap_1 = false;
boolean mainRowRejected_tMap_1 = false;
///////////////////////////////////////////////
// Starting Lookup Table "row2"
///////////////////////////////////////////////
boolean forceLooprow2 = false;
row2Struct row2ObjectFromLookup = null;
if(!rejectedInnerJoin_tMap_1) { // G_TM_M_020
hasCasePrimitiveKeyWithNull_tMap_1 = false;
row2HashKey.CustID = row1.CustID ;
row2HashKey.hashCodeDirty = true;
tHash_Lookup_row2.lookup( row2HashKey );
} // G_TM_M_020
if(tHash_Lookup_row2 != null && tHash_Lookup_row2.getCount(row2HashKey) > 1) { // G 071
//System.out.println("WARNING: UNIQUE MATCH is configured for the lookup 'row2' and it contains more one result from keys : row2.CustID = '" + row2HashKey.CustID + "'");
} // G 071
row2Struct row2 = null;
row2Struct fromLookup_row2 = null;
row2 = row2Default;
if (tHash_Lookup_row2 !=null && tHash_Lookup_row2.hasNext()) { // G 099
fromLookup_row2 = tHash_Lookup_row2.next();
} // G 099
if(fromLookup_row2 != null) {
row2 = fromLookup_row2;
}
if( !rejectedInnerJoin_tMap_1 &&
!(
context.ctxRegion.equals(row2.Region)
)
) { // G 016
fromLookup_row2 = null;
row2 = row2Default;
} // G 016
// ###############################
{ // start of Var scope
// ###############################
// # Vars tables
Var__tMap_1__Struct Var = Var__tMap_1;// ###############################
// ###############################
// # Output tables
out1 = null;
// # Output table : 'out1'
count_out1_tMap_1++;
out1_tmp.CustID = row1.CustID ;
out1_tmp.LastTransactionDate = row1.LastTransactionDate ;
out1_tmp.TotalPurchase = row1.TotalPurchase ;
out1_tmp.Region = row2.Region ;
out1_tmp.Occupation = row2.Occupation ;
out1 = out1_tmp;
log.debug("tMap_1 - Outputting the record " + count_out1_tMap_1 + " of the output table 'out1'.");
// ###############################
} // end of Var scope
rejectedInnerJoin_tMap_1 = false;
tos_count_tMap_1++;
/**
* [tMap_1 main ] stop
*/
/**
* [tMap_1 process_data_begin ] start
*/
currentComponent="tMap_1";
/**
* [tMap_1 process_data_begin ] stop
*/
// Start of branch "out1"
if(out1 != null) {
/**
* [tFileOutputDelimited_1 main ] start
*/
currentComponent="tFileOutputDelimited_1";
//out1
//out1
if(enableLogStash) {
runStat.logStatOnConnection("out1"+iterateId,1, 1);
}
if(log.isTraceEnabled()){
log.trace("out1 - " + (out1==null? "": out1.toLogString()));
}
StringBuilder sb_tFileOutputDelimited_1 = new StringBuilder();
if(out1.CustID != null) {
sb_tFileOutputDelimited_1.append(
out1.CustID
);
}
sb_tFileOutputDelimited_1.append(OUT_DELIM_tFileOutputDelimited_1);
if(out1.LastTransactionDate != null) {
sb_tFileOutputDelimited_1.append(
FormatterUtils.format_Date(out1.LastTransactionDate, "yyyy/MM/dd")
);
}
sb_tFileOutputDelimited_1.append(OUT_DELIM_tFileOutputDelimited_1);
if(out1.TotalPurchase != null) {
sb_tFileOutputDelimited_1.append(
out1.TotalPurchase
);
}
sb_tFileOutputDelimited_1.append(OUT_DELIM_tFileOutputDelimited_1);
if(out1.Region != null) {
sb_tFileOutputDelimited_1.append(
out1.Region
);
}
sb_tFileOutputDelimited_1.append(OUT_DELIM_tFileOutputDelimited_1);
if(out1.Occupation != null) {
sb_tFileOutputDelimited_1.append(
out1.Occupation
);
}
sb_tFileOutputDelimited_1.append(OUT_DELIM_ROWSEP_tFileOutputDelimited_1);
nb_line_tFileOutputDelimited_1++;
resourceMap.put("nb_line_tFileOutputDelimited_1", nb_line_tFileOutputDelimited_1);
outtFileOutputDelimited_1.write(sb_tFileOutputDelimited_1.toString());
log.debug("tFileOutputDelimited_1 - Writing the record " + nb_line_tFileOutputDelimited_1 + ".");
tos_count_tFileOutputDelimited_1++;
/**
* [tFileOutputDelimited_1 main ] stop
*/
/**
* [tFileOutputDelimited_1 process_data_begin ] start
*/
currentComponent="tFileOutputDelimited_1";
/**
* [tFileOutputDelimited_1 process_data_begin ] stop
*/
/**
* [tFileOutputDelimited_1 process_data_end ] start
*/
currentComponent="tFileOutputDelimited_1";
/**
* [tFileOutputDelimited_1 process_data_end ] stop
*/
} // End of branch "out1"
/**
* [tMap_1 process_data_end ] start
*/
currentComponent="tMap_1";
/**
* [tMap_1 process_data_end ] stop
*/
} // End of branch "row1"
/**
* [tFileInputDelimited_2 process_data_end ] start
*/
currentComponent="tFileInputDelimited_2";
/**
* [tFileInputDelimited_2 process_data_end ] stop
*/
/**
* [tFileInputDelimited_2 end ] start
*/
currentComponent="tFileInputDelimited_2";
}
}finally{
if(!((Object)("C:/Users/hryk.hoshi/Desktop/Talend/handson/Transaction_UTF8_2018Q1.csv") instanceof java.io.InputStream)){
if(fid_tFileInputDelimited_2!=null){
fid_tFileInputDelimited_2.close();
}
}
if(fid_tFileInputDelimited_2!=null){
globalMap.put("tFileInputDelimited_2_NB_LINE", fid_tFileInputDelimited_2.getRowNumber());
log.info("tFileInputDelimited_2 - Retrieved records count: "+ fid_tFileInputDelimited_2.getRowNumber() + ".");
}
}
if(log.isDebugEnabled())
log.debug("tFileInputDelimited_2 - " + ("Done.") );
ok_Hash.put("tFileInputDelimited_2", true);
end_Hash.put("tFileInputDelimited_2", System.currentTimeMillis());
/**
* [tFileInputDelimited_2 end ] stop
*/
/**
* [tMap_1 end ] start
*/
currentComponent="tMap_1";
// ###############################
// # Lookup hashes releasing
if(tHash_Lookup_row2 != null) {
tHash_Lookup_row2.endGet();
}
globalMap.remove( "tHash_Lookup_row2" );
// ###############################
log.debug("tMap_1 - Written records count in the table 'out1': " + count_out1_tMap_1 + ".");
if(enableLogStash){
if(resourceMap.get("inIterateVComp") == null || !((Boolean)resourceMap.get("inIterateVComp"))){
RunStat.StatBean talend_statebean = runStat.logStatOnConnection("row1"+iterateId,2, 0);
talendJobLog.addConnectionMessage(
"tFileInputDelimited_2",
"tFileInputDelimited",
false,
"output",
"row1",
talend_statebean.getNbLine(),
talend_statebean.getStartTime(),
talend_statebean.getEndTime()
);
talendJobLog.addConnectionMessage(
"tMap_1",
"tMap",
true,
"input",
"row1",
talend_statebean.getNbLine(),
talend_statebean.getStartTime(),
talend_statebean.getEndTime()
);
talendJobLogProcess(globalMap);
}
}
if(log.isDebugEnabled())
log.debug("tMap_1 - " + ("Done.") );
ok_Hash.put("tMap_1", true);
end_Hash.put("tMap_1", System.currentTimeMillis());
/**
* [tMap_1 end ] stop
*/
/**
* [tFileOutputDelimited_1 end ] start
*/
currentComponent="tFileOutputDelimited_1";
if(outtFileOutputDelimited_1!=null) {
outtFileOutputDelimited_1.flush();
outtFileOutputDelimited_1.close();
}
globalMap.put("tFileOutputDelimited_1_NB_LINE",nb_line_tFileOutputDelimited_1);
globalMap.put("tFileOutputDelimited_1_FILE_NAME",fileName_tFileOutputDelimited_1);
resourceMap.put("finish_tFileOutputDelimited_1", true);
log.debug("tFileOutputDelimited_1 - Written records count: " + nb_line_tFileOutputDelimited_1 + " .");
if(enableLogStash){
if(resourceMap.get("inIterateVComp") == null || !((Boolean)resourceMap.get("inIterateVComp"))){
RunStat.StatBean talend_statebean = runStat.logStatOnConnection("out1"+iterateId,2, 0);
talendJobLog.addConnectionMessage(
"tMap_1",
"tMap",
false,
"output",
"out1",
talend_statebean.getNbLine(),
talend_statebean.getStartTime(),
talend_statebean.getEndTime()
);
talendJobLog.addConnectionMessage(
"tFileOutputDelimited_1",
"tFileOutputDelimited",
true,
"input",
"out1",
talend_statebean.getNbLine(),
talend_statebean.getStartTime(),
talend_statebean.getEndTime()
);
talendJobLogProcess(globalMap);
}
}
if(log.isDebugEnabled())
log.debug("tFileOutputDelimited_1 - " + ("Done.") );
ok_Hash.put("tFileOutputDelimited_1", true);
end_Hash.put("tFileOutputDelimited_1", System.currentTimeMillis());
/**
* [tFileOutputDelimited_1 end ] stop
*/
}//end the resume
}catch(java.lang.Exception e){
if(!(e instanceof TalendException)){
log.fatal(currentComponent + " " + e.getMessage(),e);
}
TalendException te = new TalendException(e, currentComponent, globalMap);
throw te;
}catch(java.lang.Error error){
throw error;
}finally{
//free memory for "tMap_1"
globalMap.remove("tHash_Lookup_row2");
try{
/**
* [tFileInputDelimited_2 finally ] start
*/
currentComponent="tFileInputDelimited_2";
/**
* [tFileInputDelimited_2 finally ] stop
*/
/**
* [tMap_1 finally ] start
*/
currentComponent="tMap_1";
/**
* [tMap_1 finally ] stop
*/
/**
* [tFileOutputDelimited_1 finally ] start
*/
currentComponent="tFileOutputDelimited_1";
if(resourceMap.get("finish_tFileOutputDelimited_1") == null){
java.io.Writer outtFileOutputDelimited_1 = (java.io.Writer)resourceMap.get("out_tFileOutputDelimited_1");
if(outtFileOutputDelimited_1!=null) {
outtFileOutputDelimited_1.flush();
outtFileOutputDelimited_1.close();
}
}
/**
* [tFileOutputDelimited_1 finally ] stop
*/
}catch(java.lang.Exception e){
//ignore
}catch(java.lang.Error error){
//ignore
}
resourceMap = null;
}
globalMap.put("tFileInputDelimited_2_SUBPROCESS_STATE", 1);
}
public static class row2Struct implements routines.system.IPersistableComparableLookupRow<row2Struct> {
final static byte[] commonByteArrayLock_LOCAL_PROJECT_Job4 = new byte[0];
static byte[] commonByteArray_LOCAL_PROJECT_Job4 = new byte[0];
protected static final int DEFAULT_HASHCODE = 1;
protected static final int PRIME = 31;
protected int hashCode = DEFAULT_HASHCODE;
public boolean hashCodeDirty = true;
public String loopKey;
public Integer CustID;
public Integer getCustID () {
return this.CustID;
}
public String Last_Name_Kanji;
public String getLast_Name_Kanji () {
return this.Last_Name_Kanji;
}
public String Last_Name_Kana;
public String getLast_Name_Kana () {
return this.Last_Name_Kana;
}
public String First_Name_Kanji;
public String getFirst_Name_Kanji () {
return this.First_Name_Kanji;
}
public String First_Name_Kana;
public String getFirst_Name_Kana () {
return this.First_Name_Kana;
}
public String Gender;
public String getGender () {
return this.Gender;
}
public String Pref;
public String getPref () {
return this.Pref;
}
public String Address_1;
public String getAddress_1 () {
return this.Address_1;
}
public String Address_2;
public String getAddress_2 () {
return this.Address_2;
}
public String Postal_Code;
public String getPostal_Code () {
return this.Postal_Code;
}
public java.util.Date Phone;
public java.util.Date getPhone () {
return this.Phone;
}
public String My_Number;
public String getMy_Number () {
return this.My_Number;
}
public String Birth;
public String getBirth () {
return this.Birth;
}
public String Region;
public String getRegion () {
return this.Region;
}
public String Occupation;
public String getOccupation () {
return this.Occupation;
}
public String Industry;
public String getIndustry () {
return this.Industry;
}
public String Company;
public String getCompany () {
return this.Company;
}
@Override
public int hashCode() {
if (this.hashCodeDirty) {
final int prime = PRIME;
int result = DEFAULT_HASHCODE;
result = prime * result + ((this.CustID == null) ? 0 : this.CustID.hashCode());
this.hashCode = result;
this.hashCodeDirty = false;
}
return this.hashCode;
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
final row2Struct other = (row2Struct) obj;
if (this.CustID == null) {
if (other.CustID != null)
return false;
} else if (!this.CustID.equals(other.CustID))
return false;
return true;
}
public void copyDataTo(row2Struct other) {
other.CustID = this.CustID;
other.Last_Name_Kanji = this.Last_Name_Kanji;
other.Last_Name_Kana = this.Last_Name_Kana;
other.First_Name_Kanji = this.First_Name_Kanji;
other.First_Name_Kana = this.First_Name_Kana;
other.Gender = this.Gender;
other.Pref = this.Pref;
other.Address_1 = this.Address_1;
other.Address_2 = this.Address_2;
other.Postal_Code = this.Postal_Code;
other.Phone = this.Phone;
other.My_Number = this.My_Number;
other.Birth = this.Birth;
other.Region = this.Region;
other.Occupation = this.Occupation;
other.Industry = this.Industry;
other.Company = this.Company;
}
public void copyKeysDataTo(row2Struct other) {
other.CustID = this.CustID;
}
private Integer readInteger(ObjectInputStream dis) throws IOException{
Integer intReturn;
int length = 0;
length = dis.readByte();
if (length == -1) {
intReturn = null;
} else {
intReturn = dis.readInt();
}
return intReturn;
}
private void writeInteger(Integer intNum, ObjectOutputStream dos) throws IOException{
if(intNum == null) {
dos.writeByte(-1);
} else {
dos.writeByte(0);
dos.writeInt(intNum);
}
}
private String readString(DataInputStream dis, ObjectInputStream ois) throws IOException{
String strReturn = null;
int length = 0;
length = dis.readInt();
if (length == -1) {
strReturn = null;
} else {
byte[] byteArray = new byte[length];
dis.read(byteArray);
strReturn = new String(byteArray, utf8Charset);
}
return strReturn;
}
private void writeString(String str, DataOutputStream dos, ObjectOutputStream oos) throws IOException{
if(str == null) {
dos.writeInt(-1);
} else {
byte[] byteArray = str.getBytes(utf8Charset);
dos.writeInt(byteArray.length);
dos.write(byteArray);
}
}
private java.util.Date readDate(DataInputStream dis, ObjectInputStream ois) throws IOException{
java.util.Date dateReturn = null;
int length = 0;
length = dis.readByte();
if (length == -1) {
dateReturn = null;
} else {
dateReturn = new Date(dis.readLong());
}
return dateReturn;
}
private void writeDate(java.util.Date date1, DataOutputStream dos, ObjectOutputStream oos) throws IOException{
if(date1 == null) {
dos.writeByte(-1);
} else {
dos.writeByte(0);
dos.writeLong(date1.getTime());
}
}
public void readKeysData(ObjectInputStream dis) {
synchronized(commonByteArrayLock_LOCAL_PROJECT_Job4) {
try {
int length = 0;
this.CustID = readInteger(dis);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
public void writeKeysData(ObjectOutputStream dos) {
try {
// Integer
writeInteger(this.CustID,dos);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Fill Values data by reading ObjectInputStream.
*/
public void readValuesData(DataInputStream dis, ObjectInputStream ois) {
try {
int length = 0;
this.Last_Name_Kanji = readString(dis,ois);
this.Last_Name_Kana = readString(dis,ois);
this.First_Name_Kanji = readString(dis,ois);
this.First_Name_Kana = readString(dis,ois);
this.Gender = readString(dis,ois);
this.Pref = readString(dis,ois);
this.Address_1 = readString(dis,ois);
this.Address_2 = readString(dis,ois);
this.Postal_Code = readString(dis,ois);
this.Phone = readDate(dis,ois);
this.My_Number = readString(dis,ois);
this.Birth = readString(dis,ois);
this.Region = readString(dis,ois);
this.Occupation = readString(dis,ois);
this.Industry = readString(dis,ois);
this.Company = readString(dis,ois);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Return a byte array which represents Values data.
*/
public void writeValuesData(DataOutputStream dos, ObjectOutputStream oos) {
try {
writeString(this.Last_Name_Kanji, dos, oos);
writeString(this.Last_Name_Kana, dos, oos);
writeString(this.First_Name_Kanji, dos, oos);
writeString(this.First_Name_Kana, dos, oos);
writeString(this.Gender, dos, oos);
writeString(this.Pref, dos, oos);
writeString(this.Address_1, dos, oos);
writeString(this.Address_2, dos, oos);
writeString(this.Postal_Code, dos, oos);
writeDate(this.Phone, dos, oos);
writeString(this.My_Number, dos, oos);
writeString(this.Birth, dos, oos);
writeString(this.Region, dos, oos);
writeString(this.Occupation, dos, oos);
writeString(this.Industry, dos, oos);
writeString(this.Company, dos, oos);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(super.toString());
sb.append("[");
sb.append("CustID="+String.valueOf(CustID));
sb.append(",Last_Name_Kanji="+Last_Name_Kanji);
sb.append(",Last_Name_Kana="+Last_Name_Kana);
sb.append(",First_Name_Kanji="+First_Name_Kanji);
sb.append(",First_Name_Kana="+First_Name_Kana);
sb.append(",Gender="+Gender);
sb.append(",Pref="+Pref);
sb.append(",Address_1="+Address_1);
sb.append(",Address_2="+Address_2);
sb.append(",Postal_Code="+Postal_Code);
sb.append(",Phone="+String.valueOf(Phone));
sb.append(",My_Number="+My_Number);
sb.append(",Birth="+Birth);
sb.append(",Region="+Region);
sb.append(",Occupation="+Occupation);
sb.append(",Industry="+Industry);
sb.append(",Company="+Company);
sb.append("]");
return sb.toString();
}
public String toLogString(){
StringBuilder sb = new StringBuilder();
if(CustID == null){
sb.append("<null>");
}else{
sb.append(CustID);
}
sb.append("|");
if(Last_Name_Kanji == null){
sb.append("<null>");
}else{
sb.append(Last_Name_Kanji);
}
sb.append("|");
if(Last_Name_Kana == null){
sb.append("<null>");
}else{
sb.append(Last_Name_Kana);
}
sb.append("|");
if(First_Name_Kanji == null){
sb.append("<null>");
}else{
sb.append(First_Name_Kanji);
}
sb.append("|");
if(First_Name_Kana == null){
sb.append("<null>");
}else{
sb.append(First_Name_Kana);
}
sb.append("|");
if(Gender == null){
sb.append("<null>");
}else{
sb.append(Gender);
}
sb.append("|");
if(Pref == null){
sb.append("<null>");
}else{
sb.append(Pref);
}
sb.append("|");
if(Address_1 == null){
sb.append("<null>");
}else{
sb.append(Address_1);
}
sb.append("|");
if(Address_2 == null){
sb.append("<null>");
}else{
sb.append(Address_2);
}
sb.append("|");
if(Postal_Code == null){
sb.append("<null>");
}else{
sb.append(Postal_Code);
}
sb.append("|");
if(Phone == null){
sb.append("<null>");
}else{
sb.append(Phone);
}
sb.append("|");
if(My_Number == null){
sb.append("<null>");
}else{
sb.append(My_Number);
}
sb.append("|");
if(Birth == null){
sb.append("<null>");
}else{
sb.append(Birth);
}
sb.append("|");
if(Region == null){
sb.append("<null>");
}else{
sb.append(Region);
}
sb.append("|");
if(Occupation == null){
sb.append("<null>");
}else{
sb.append(Occupation);
}
sb.append("|");
if(Industry == null){
sb.append("<null>");
}else{
sb.append(Industry);
}
sb.append("|");
if(Company == null){
sb.append("<null>");
}else{
sb.append(Company);
}
sb.append("|");
return sb.toString();
}
/**
* Compare keys
*/
public int compareTo(row2Struct other) {
int returnValue = -1;
returnValue = checkNullsAndCompare(this.CustID, other.CustID);
if(returnValue != 0) {
return returnValue;
}
return returnValue;
}
private int checkNullsAndCompare(Object object1, Object object2) {
int returnValue = 0;
if (object1 instanceof Comparable && object2 instanceof Comparable) {
returnValue = ((Comparable) object1).compareTo(object2);
} else if (object1 != null && object2 != null) {
returnValue = compareStrings(object1.toString(), object2.toString());
} else if (object1 == null && object2 != null) {
returnValue = 1;
} else if (object1 != null && object2 == null) {
returnValue = -1;
} else {
returnValue = 0;
}
return returnValue;
}
private int compareStrings(String string1, String string2) {
return string1.compareTo(string2);
}
}
public void tFileInputDelimited_1Process(final java.util.Map<String, Object> globalMap) throws TalendException {
globalMap.put("tFileInputDelimited_1_SUBPROCESS_STATE", 0);
final boolean execStat = this.execStat;
String iterateId = "";
String currentComponent = "";
java.util.Map<String, Object> resourceMap = new java.util.HashMap<String, Object>();
try {
// TDI-39566 avoid throwing an useless Exception
boolean resumeIt = true;
if (globalResumeTicket == false && resumeEntryMethodName != null) {
String currentMethodName = new java.lang.Exception().getStackTrace()[0].getMethodName();
resumeIt = resumeEntryMethodName.equals(currentMethodName);
}
if (resumeIt || globalResumeTicket) { //start the resume
globalResumeTicket = true;
row2Struct row2 = new row2Struct();
/**
* [tAdvancedHash_row2 begin ] start
*/
ok_Hash.put("tAdvancedHash_row2", false);
start_Hash.put("tAdvancedHash_row2", System.currentTimeMillis());
currentComponent="tAdvancedHash_row2";
if (enableLogStash) {
if(resourceMap.get("inIterateVComp") == null){
if(enableLogStash) {
runStat.logStatOnConnection("row2"+iterateId, 0, 0);
}
}
}
int tos_count_tAdvancedHash_row2 = 0;
if(enableLogStash) {
talendJobLog.addComponentMessage("tAdvancedHash_row2", "tAdvancedHash");
talendJobLogProcess(globalMap);
}
// connection name:row2
// source node:tFileInputDelimited_1 - inputs:(after_tFileInputDelimited_2) outputs:(row2,row2) | target node:tAdvancedHash_row2 - inputs:(row2) outputs:()
// linked node: tMap_1 - inputs:(row1,row2) outputs:(out1)
org.talend.designer.components.lookup.common.ICommonLookup.MATCHING_MODE matchingModeEnum_row2 =
org.talend.designer.components.lookup.common.ICommonLookup.MATCHING_MODE.UNIQUE_MATCH;
org.talend.designer.components.lookup.memory.AdvancedMemoryLookup<row2Struct> tHash_Lookup_row2 =org.talend.designer.components.lookup.memory.AdvancedMemoryLookup.
<row2Struct>getLookup(matchingModeEnum_row2);
globalMap.put("tHash_Lookup_row2", tHash_Lookup_row2);
/**
* [tAdvancedHash_row2 begin ] stop
*/
/**
* [tFileInputDelimited_1 begin ] start
*/
ok_Hash.put("tFileInputDelimited_1", false);
start_Hash.put("tFileInputDelimited_1", System.currentTimeMillis());
currentComponent="tFileInputDelimited_1";
int tos_count_tFileInputDelimited_1 = 0;
if(log.isDebugEnabled())
log.debug("tFileInputDelimited_1 - " + ("Start to work.") );
if (log.isDebugEnabled()) {
class BytesLimit65535_tFileInputDelimited_1{
public void limitLog4jByte() throws Exception{
StringBuilder log4jParamters_tFileInputDelimited_1 = new StringBuilder();
log4jParamters_tFileInputDelimited_1.append("Parameters:");
log4jParamters_tFileInputDelimited_1.append("FILENAME" + " = " + "\"C:/Users/hryk.hoshi/Desktop/Talend/handson/JCustomer_UTF8_2018.csv\"");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("CSV_OPTION" + " = " + "false");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("ROWSEPARATOR" + " = " + "\"\\n\"");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("FIELDSEPARATOR" + " = " + "\",\"");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("HEADER" + " = " + "1");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("FOOTER" + " = " + "0");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("LIMIT" + " = " + "");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("REMOVE_EMPTY_ROW" + " = " + "false");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("UNCOMPRESS" + " = " + "false");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("DIE_ON_ERROR" + " = " + "false");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("ADVANCED_SEPARATOR" + " = " + "false");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("RANDOM" + " = " + "false");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("TRIMALL" + " = " + "false");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("TRIMSELECT" + " = " + "[{TRIM="+("false")+", SCHEMA_COLUMN="+("CustID")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Last_Name_Kanji")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Last_Name_Kana")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("First_Name_Kanji")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("First_Name_Kana")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Gender")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Pref")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Address_1")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Address_2")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Postal_Code")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Phone")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("My_Number")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Birth")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Region")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Occupation")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Industry")+"}, {TRIM="+("false")+", SCHEMA_COLUMN="+("Company")+"}]");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("CHECK_FIELDS_NUM" + " = " + "false");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("CHECK_DATE" + " = " + "false");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("ENCODING" + " = " + "\"UTF-8\"");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("SPLITRECORD" + " = " + "false");
log4jParamters_tFileInputDelimited_1.append(" | ");
log4jParamters_tFileInputDelimited_1.append("ENABLE_DECODE" + " = " + "false");
log4jParamters_tFileInputDelimited_1.append(" | ");
if(log.isDebugEnabled())
log.debug("tFileInputDelimited_1 - " + (log4jParamters_tFileInputDelimited_1) );
}
}
new BytesLimit65535_tFileInputDelimited_1().limitLog4jByte();
}
if(enableLogStash) {
talendJobLog.addComponentMessage("tFileInputDelimited_1", "tFileInputDelimited");
talendJobLogProcess(globalMap);
}
final routines.system.RowState rowstate_tFileInputDelimited_1 = new routines.system.RowState();
int nb_line_tFileInputDelimited_1 = 0;
org.talend.fileprocess.FileInputDelimited fid_tFileInputDelimited_1 = null;
int limit_tFileInputDelimited_1 = -1;
try{
Object filename_tFileInputDelimited_1 = "C:/Users/hryk.hoshi/Desktop/Talend/handson/JCustomer_UTF8_2018.csv";
if(filename_tFileInputDelimited_1 instanceof java.io.InputStream){
int footer_value_tFileInputDelimited_1 = 0, random_value_tFileInputDelimited_1 = -1;
if(footer_value_tFileInputDelimited_1 >0 || random_value_tFileInputDelimited_1 > 0){
throw new java.lang.Exception("When the input source is a stream,footer and random shouldn't be bigger than 0.");
}
}
try {
fid_tFileInputDelimited_1 = new org.talend.fileprocess.FileInputDelimited("C:/Users/hryk.hoshi/Desktop/Talend/handson/JCustomer_UTF8_2018.csv", "UTF-8",",","\n",false,1,0,
limit_tFileInputDelimited_1
,-1, false);
} catch(java.lang.Exception e) {
log.error("tFileInputDelimited_1 - " +e.getMessage());
System.err.println(e.getMessage());
}
log.info("tFileInputDelimited_1 - Retrieving records from the datasource.");
while (fid_tFileInputDelimited_1!=null && fid_tFileInputDelimited_1.nextRecord()) {
rowstate_tFileInputDelimited_1.reset();
row2 = null;
row2 = null;
boolean whetherReject_tFileInputDelimited_1 = false;
row2 = new row2Struct();
try {
int columnIndexWithD_tFileInputDelimited_1 = 0;
String temp = "";
columnIndexWithD_tFileInputDelimited_1 = 0;
temp = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
if(temp.length() > 0) {
try {
row2.CustID = ParserUtils.parseTo_Integer(temp);
} catch(java.lang.Exception ex_tFileInputDelimited_1) {
rowstate_tFileInputDelimited_1.setException(new RuntimeException(String.format("Couldn't parse value for column '%s' in '%s', value is '%s'. Details: %s",
"CustID", "row2", temp, ex_tFileInputDelimited_1), ex_tFileInputDelimited_1));
}
} else {
row2.CustID = null;
}
columnIndexWithD_tFileInputDelimited_1 = 1;
row2.Last_Name_Kanji = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 2;
row2.Last_Name_Kana = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 3;
row2.First_Name_Kanji = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 4;
row2.First_Name_Kana = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 5;
row2.Gender = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 6;
row2.Pref = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 7;
row2.Address_1 = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 8;
row2.Address_2 = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 9;
row2.Postal_Code = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 10;
temp = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
if(temp.length() > 0) {
try {
row2.Phone = ParserUtils.parseTo_Date(temp, "dd-MM-yyyy");
} catch(java.lang.Exception ex_tFileInputDelimited_1) {
rowstate_tFileInputDelimited_1.setException(new RuntimeException(String.format("Couldn't parse value for column '%s' in '%s', value is '%s'. Details: %s",
"Phone", "row2", temp, ex_tFileInputDelimited_1), ex_tFileInputDelimited_1));
}
} else {
row2.Phone = null;
}
columnIndexWithD_tFileInputDelimited_1 = 11;
row2.My_Number = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 12;
row2.Birth = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 13;
row2.Region = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 14;
row2.Occupation = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 15;
row2.Industry = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
columnIndexWithD_tFileInputDelimited_1 = 16;
row2.Company = fid_tFileInputDelimited_1.get(columnIndexWithD_tFileInputDelimited_1);
if(rowstate_tFileInputDelimited_1.getException()!=null) {
throw rowstate_tFileInputDelimited_1.getException();
}
} catch (java.lang.Exception e) {
whetherReject_tFileInputDelimited_1 = true;
log.error("tFileInputDelimited_1 - " +e.getMessage());
System.err.println(e.getMessage());
row2 = null;
}
log.debug("tFileInputDelimited_1 - Retrieving the record " + fid_tFileInputDelimited_1.getRowNumber() + ".");
/**
* [tFileInputDelimited_1 begin ] stop
*/
/**
* [tFileInputDelimited_1 main ] start
*/
currentComponent="tFileInputDelimited_1";
tos_count_tFileInputDelimited_1++;
/**
* [tFileInputDelimited_1 main ] stop
*/
/**
* [tFileInputDelimited_1 process_data_begin ] start
*/
currentComponent="tFileInputDelimited_1";
/**
* [tFileInputDelimited_1 process_data_begin ] stop
*/
// Start of branch "row2"
if(row2 != null) {
/**
* [tAdvancedHash_row2 main ] start
*/
currentComponent="tAdvancedHash_row2";
//row2
//row2
if(enableLogStash) {
runStat.logStatOnConnection("row2"+iterateId,1, 1);
}
if(log.isTraceEnabled()){
log.trace("row2 - " + (row2==null? "": row2.toLogString()));
}
row2Struct row2_HashRow = new row2Struct();
row2_HashRow.CustID = row2.CustID;
row2_HashRow.Last_Name_Kanji = row2.Last_Name_Kanji;
row2_HashRow.Last_Name_Kana = row2.Last_Name_Kana;
row2_HashRow.First_Name_Kanji = row2.First_Name_Kanji;
row2_HashRow.First_Name_Kana = row2.First_Name_Kana;
row2_HashRow.Gender = row2.Gender;
row2_HashRow.Pref = row2.Pref;
row2_HashRow.Address_1 = row2.Address_1;
row2_HashRow.Address_2 = row2.Address_2;
row2_HashRow.Postal_Code = row2.Postal_Code;
row2_HashRow.Phone = row2.Phone;
row2_HashRow.My_Number = row2.My_Number;
row2_HashRow.Birth = row2.Birth;
row2_HashRow.Region = row2.Region;
row2_HashRow.Occupation = row2.Occupation;
row2_HashRow.Industry = row2.Industry;
row2_HashRow.Company = row2.Company;
tHash_Lookup_row2.put(row2_HashRow);
tos_count_tAdvancedHash_row2++;
/**
* [tAdvancedHash_row2 main ] stop
*/
/**
* [tAdvancedHash_row2 process_data_begin ] start
*/
currentComponent="tAdvancedHash_row2";
/**
* [tAdvancedHash_row2 process_data_begin ] stop
*/
/**
* [tAdvancedHash_row2 process_data_end ] start
*/
currentComponent="tAdvancedHash_row2";
/**
* [tAdvancedHash_row2 process_data_end ] stop
*/
} // End of branch "row2"
/**
* [tFileInputDelimited_1 process_data_end ] start
*/
currentComponent="tFileInputDelimited_1";
/**
* [tFileInputDelimited_1 process_data_end ] stop
*/
/**
* [tFileInputDelimited_1 end ] start
*/
currentComponent="tFileInputDelimited_1";
}
}finally{
if(!((Object)("C:/Users/hryk.hoshi/Desktop/Talend/handson/JCustomer_UTF8_2018.csv") instanceof java.io.InputStream)){
if(fid_tFileInputDelimited_1!=null){
fid_tFileInputDelimited_1.close();
}
}
if(fid_tFileInputDelimited_1!=null){
globalMap.put("tFileInputDelimited_1_NB_LINE", fid_tFileInputDelimited_1.getRowNumber());
log.info("tFileInputDelimited_1 - Retrieved records count: "+ fid_tFileInputDelimited_1.getRowNumber() + ".");
}
}
if(log.isDebugEnabled())
log.debug("tFileInputDelimited_1 - " + ("Done.") );
ok_Hash.put("tFileInputDelimited_1", true);
end_Hash.put("tFileInputDelimited_1", System.currentTimeMillis());
/**
* [tFileInputDelimited_1 end ] stop
*/
/**
* [tAdvancedHash_row2 end ] start
*/
currentComponent="tAdvancedHash_row2";
tHash_Lookup_row2.endPut();
if(enableLogStash){
if(resourceMap.get("inIterateVComp") == null || !((Boolean)resourceMap.get("inIterateVComp"))){
RunStat.StatBean talend_statebean = runStat.logStatOnConnection("row2"+iterateId,2, 0);
talendJobLog.addConnectionMessage(
"tFileInputDelimited_1",
"tFileInputDelimited",
false,
"output",
"row2",
talend_statebean.getNbLine(),
talend_statebean.getStartTime(),
talend_statebean.getEndTime()
);
talendJobLog.addConnectionMessage(
"tAdvancedHash_row2",
"tAdvancedHash",
true,
"input",
"row2",
talend_statebean.getNbLine(),
talend_statebean.getStartTime(),
talend_statebean.getEndTime()
);
talendJobLogProcess(globalMap);
}
}
ok_Hash.put("tAdvancedHash_row2", true);
end_Hash.put("tAdvancedHash_row2", System.currentTimeMillis());
/**
* [tAdvancedHash_row2 end ] stop
*/
}//end the resume
}catch(java.lang.Exception e){
if(!(e instanceof TalendException)){
log.fatal(currentComponent + " " + e.getMessage(),e);
}
TalendException te = new TalendException(e, currentComponent, globalMap);
throw te;
}catch(java.lang.Error error){
throw error;
}finally{
try{
/**
* [tFileInputDelimited_1 finally ] start
*/
currentComponent="tFileInputDelimited_1";
/**
* [tFileInputDelimited_1 finally ] stop
*/
/**
* [tAdvancedHash_row2 finally ] start
*/
currentComponent="tAdvancedHash_row2";
/**
* [tAdvancedHash_row2 finally ] stop
*/
}catch(java.lang.Exception e){
//ignore
}catch(java.lang.Error error){
//ignore
}
resourceMap = null;
}
globalMap.put("tFileInputDelimited_1_SUBPROCESS_STATE", 1);
}
public void talendJobLogProcess(final java.util.Map<String, Object> globalMap) throws TalendException {
globalMap.put("talendJobLog_SUBPROCESS_STATE", 0);
final boolean execStat = this.execStat;
String iterateId = "";
String currentComponent = "";
java.util.Map<String, Object> resourceMap = new java.util.HashMap<String, Object>();
try {
// TDI-39566 avoid throwing an useless Exception
boolean resumeIt = true;
if (globalResumeTicket == false && resumeEntryMethodName != null) {
String currentMethodName = new java.lang.Exception().getStackTrace()[0].getMethodName();
resumeIt = resumeEntryMethodName.equals(currentMethodName);
}
if (resumeIt || globalResumeTicket) { //start the resume
globalResumeTicket = true;
/**
* [talendJobLog begin ] start
*/
ok_Hash.put("talendJobLog", false);
start_Hash.put("talendJobLog", System.currentTimeMillis());
currentComponent="talendJobLog";
int tos_count_talendJobLog = 0;
for (JobStructureCatcherUtils.JobStructureCatcherMessage jcm : talendJobLog.getMessages()) {
org.talend.logging.audit.Context log_context_talendJobLog = null;
if(jcm.component_name == null) {//job level log
if(jcm.status == null) {//job start
log_context_talendJobLog = org.talend.job.audit.JobContextBuilder.create().jobName(jcm.job_name).jobId(jcm.job_id).jobVersion(jcm.job_version).timestamp(jcm.moment).build();
auditLogger_talendJobLog.jobstart(log_context_talendJobLog);
} else {//job end
long timeMS = jcm.end_time - jcm.start_time;
String duration = String.format(java.util.Locale.US, "%1$.2fs", (timeMS * 1.0)/1000);
log_context_talendJobLog = org.talend.job.audit.JobContextBuilder.create().jobName(jcm.job_name).jobId(jcm.job_id).jobVersion(jcm.job_version)
.timestamp(jcm.moment).duration(duration).status(jcm.status).build();
auditLogger_talendJobLog.jobstop(log_context_talendJobLog);
}
} else if(jcm.current_connector == null) {//component log
log_context_talendJobLog = org.talend.job.audit.JobContextBuilder.create().jobName(jcm.job_name).jobId(jcm.job_id).jobVersion(jcm.job_version)
.connectorType(jcm.component_name).connectorId(jcm.component_id).build();
auditLogger_talendJobLog.runcomponent(log_context_talendJobLog);
} else {//component connector meter log
long timeMS = jcm.end_time - jcm.start_time;
String duration = String.format(java.util.Locale.US, "%1$.2fs", (timeMS * 1.0)/1000);
if(jcm.current_connector_as_input) {//log current component input line
log_context_talendJobLog = org.talend.job.audit.JobContextBuilder.create().jobName(jcm.job_name).jobId(jcm.job_id).jobVersion(jcm.job_version)
.connectorType(jcm.component_name).connectorId(jcm.component_id)
.connectionName(jcm.current_connector).connectionType(jcm.current_connector_type)
.rows(jcm.total_row_number).duration(duration).build();
auditLogger_talendJobLog.flowInput(log_context_talendJobLog);
} else {//log current component output/reject line
log_context_talendJobLog = org.talend.job.audit.JobContextBuilder.create().jobName(jcm.job_name).jobId(jcm.job_id).jobVersion(jcm.job_version)
.connectorType(jcm.component_name).connectorId(jcm.component_id)
.connectionName(jcm.current_connector).connectionType(jcm.current_connector_type)
.rows(jcm.total_row_number).duration(duration).build();
auditLogger_talendJobLog.flowOutput(log_context_talendJobLog);
}
}
}
/**
* [talendJobLog begin ] stop
*/
/**
* [talendJobLog main ] start
*/
currentComponent="talendJobLog";
tos_count_talendJobLog++;
/**
* [talendJobLog main ] stop
*/
/**
* [talendJobLog process_data_begin ] start
*/
currentComponent="talendJobLog";
/**
* [talendJobLog process_data_begin ] stop
*/
/**
* [talendJobLog process_data_end ] start
*/
currentComponent="talendJobLog";
/**
* [talendJobLog process_data_end ] stop
*/
/**
* [talendJobLog end ] start
*/
currentComponent="talendJobLog";
ok_Hash.put("talendJobLog", true);
end_Hash.put("talendJobLog", System.currentTimeMillis());
/**
* [talendJobLog end ] stop
*/
}//end the resume
}catch(java.lang.Exception e){
if(!(e instanceof TalendException)){
log.fatal(currentComponent + " " + e.getMessage(),e);
}
TalendException te = new TalendException(e, currentComponent, globalMap);
throw te;
}catch(java.lang.Error error){
throw error;
}finally{
try{
/**
* [talendJobLog finally ] start
*/
currentComponent="talendJobLog";
/**
* [talendJobLog finally ] stop
*/
}catch(java.lang.Exception e){
//ignore
}catch(java.lang.Error error){
//ignore
}
resourceMap = null;
}
globalMap.put("talendJobLog_SUBPROCESS_STATE", 1);
}
public String resuming_logs_dir_path = null;
public String resuming_checkpoint_path = null;
public String parent_part_launcher = null;
private String resumeEntryMethodName = null;
private boolean globalResumeTicket = false;
public boolean watch = false;
// portStats is null, it means don't execute the statistics
public Integer portStats = null;
public int portTraces = 4334;
public String clientHost;
public String defaultClientHost = "localhost";
public String contextStr = "Default";
public boolean isDefaultContext = true;
public String pid = "0";
public String rootPid = null;
public String fatherPid = null;
public String fatherNode = null;
public long startTime = 0;
public boolean isChildJob = false;
public String log4jLevel = "";
private boolean enableLogStash;
private boolean execStat = true;
private ThreadLocal<java.util.Map<String, String>> threadLocal = new ThreadLocal<java.util.Map<String, String>>() {
protected java.util.Map<String, String> initialValue() {
java.util.Map<String,String> threadRunResultMap = new java.util.HashMap<String, String>();
threadRunResultMap.put("errorCode", null);
threadRunResultMap.put("status", "");
return threadRunResultMap;
};
};
private PropertiesWithType context_param = new PropertiesWithType();
public java.util.Map<String, Object> parentContextMap = new java.util.HashMap<String, Object>();
public String status= "";
public static void main(String[] args){
final Job4 Job4Class = new Job4();
int exitCode = Job4Class.runJobInTOS(args);
if(exitCode==0){
log.info("TalendJob: 'Job4' - Done.");
}
System.exit(exitCode);
}
public String[][] runJob(String[] args) {
int exitCode = runJobInTOS(args);
String[][] bufferValue = new String[][] { { Integer.toString(exitCode) } };
return bufferValue;
}
public boolean hastBufferOutputComponent() {
boolean hastBufferOutput = false;
return hastBufferOutput;
}
public int runJobInTOS(String[] args) {
// reset status
status = "";
String lastStr = "";
for (String arg : args) {
if (arg.equalsIgnoreCase("--context_param")) {
lastStr = arg;
} else if (lastStr.equals("")) {
evalParam(arg);
} else {
evalParam(lastStr + " " + arg);
lastStr = "";
}
}
if(!"".equals(log4jLevel)){
if("trace".equalsIgnoreCase(log4jLevel)){
log.setLevel(org.apache.log4j.Level.TRACE);
}else if("debug".equalsIgnoreCase(log4jLevel)){
log.setLevel(org.apache.log4j.Level.DEBUG);
}else if("info".equalsIgnoreCase(log4jLevel)){
log.setLevel(org.apache.log4j.Level.INFO);
}else if("warn".equalsIgnoreCase(log4jLevel)){
log.setLevel(org.apache.log4j.Level.WARN);
}else if("error".equalsIgnoreCase(log4jLevel)){
log.setLevel(org.apache.log4j.Level.ERROR);
}else if("fatal".equalsIgnoreCase(log4jLevel)){
log.setLevel(org.apache.log4j.Level.FATAL);
}else if ("off".equalsIgnoreCase(log4jLevel)){
log.setLevel(org.apache.log4j.Level.OFF);
}
org.apache.log4j.Logger.getRootLogger().setLevel(log.getLevel());
}
log.info("TalendJob: 'Job4' - Start.");
if(enableLogStash) {
java.util.Properties properties_talendJobLog = new java.util.Properties();
properties_talendJobLog.setProperty("root.logger", "audit");
properties_talendJobLog.setProperty("encoding", "UTF-8");
properties_talendJobLog.setProperty("application.name", "Talend Studio");
properties_talendJobLog.setProperty("service.name", "Talend Studio Job");
properties_talendJobLog.setProperty("instance.name", "Talend Studio Job Instance");
properties_talendJobLog.setProperty("propagate.appender.exceptions", "none");
properties_talendJobLog.setProperty("log.appender", "file");
properties_talendJobLog.setProperty("appender.file.path", "audit.json");
properties_talendJobLog.setProperty("appender.file.maxsize", "52428800");
properties_talendJobLog.setProperty("appender.file.maxbackup", "20");
properties_talendJobLog.setProperty("host", "false");
auditLogger_talendJobLog = org.talend.job.audit.JobEventAuditLoggerFactory.createJobAuditLogger(properties_talendJobLog);
}
if(clientHost == null) {
clientHost = defaultClientHost;
}
if(pid == null || "0".equals(pid)) {
pid = TalendString.getAsciiRandomString(6);
}
if (rootPid==null) {
rootPid = pid;
}
if (fatherPid==null) {
fatherPid = pid;
}else{
isChildJob = true;
}
try {
//call job/subjob with an existing context, like: --context=production. if without this parameter, there will use the default context instead.
java.io.InputStream inContext = Job4.class.getClassLoader().getResourceAsStream("local_project/job4_0_1/contexts/" + contextStr + ".properties");
if (inContext == null) {
inContext = Job4.class.getClassLoader().getResourceAsStream("config/contexts/" + contextStr + ".properties");
}
if (inContext != null) {
//defaultProps is in order to keep the original context value
if(context != null && context.isEmpty()) {
defaultProps.load(inContext);
context = new ContextProperties(defaultProps);
}
inContext.close();
} else if (!isDefaultContext) {
//print info and job continue to run, for case: context_param is not empty.
System.err.println("Could not find the context " + contextStr);
}
if(!context_param.isEmpty()) {
context.putAll(context_param);
//set types for params from parentJobs
for (Object key: context_param.keySet()){
String context_key = key.toString();
String context_type = context_param.getContextType(context_key);
context.setContextType(context_key, context_type);
}
}
class ContextProcessing {
private void processContext_0() {
context.setContextType("ctxRegion", "id_String");
context.ctxRegion=(String) context.getProperty("ctxRegion");
}
public void processAllContext() {
processContext_0();
}
}
new ContextProcessing().processAllContext();
} catch (java.io.IOException ie) {
System.err.println("Could not load context "+contextStr);
ie.printStackTrace();
}
// get context value from parent directly
if (parentContextMap != null && !parentContextMap.isEmpty()) {if (parentContextMap.containsKey("ctxRegion")) {
context.ctxRegion = (String) parentContextMap.get("ctxRegion");
}
}
//Resume: init the resumeUtil
resumeEntryMethodName = ResumeUtil.getResumeEntryMethodName(resuming_checkpoint_path);
resumeUtil = new ResumeUtil(resuming_logs_dir_path, isChildJob, rootPid);
resumeUtil.initCommonInfo(pid, rootPid, fatherPid, projectName, jobName, contextStr, jobVersion);
List<String> parametersToEncrypt = new java.util.ArrayList<String>();
//Resume: jobStart
resumeUtil.addLog("JOB_STARTED", "JOB:" + jobName, parent_part_launcher, Thread.currentThread().getId() + "", "","","","",resumeUtil.convertToJsonText(context,parametersToEncrypt));
java.util.concurrent.ConcurrentHashMap<Object, Object> concurrentHashMap = new java.util.concurrent.ConcurrentHashMap<Object, Object>();
globalMap.put("concurrentHashMap", concurrentHashMap);
long startUsedMemory = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
long endUsedMemory = 0;
long end = 0;
startTime = System.currentTimeMillis();
this.globalResumeTicket = true;//to run tPreJob
if(enableLogStash) {
talendJobLog.addJobStartMessage();
try {
talendJobLogProcess(globalMap);
} catch (java.lang.Exception e) {
e.printStackTrace();
}
}
this.globalResumeTicket = false;//to run others jobs
try {
errorCode = null;tFileInputDelimited_2Process(globalMap);
if(!"failure".equals(status)) { status = "end"; }
}catch (TalendException e_tFileInputDelimited_2) {
globalMap.put("tFileInputDelimited_2_SUBPROCESS_STATE", -1);
e_tFileInputDelimited_2.printStackTrace();
}
this.globalResumeTicket = true;//to run tPostJob
end = System.currentTimeMillis();
if (watch) {
System.out.println((end-startTime)+" milliseconds");
}
endUsedMemory = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
if (false) {
System.out.println((endUsedMemory - startUsedMemory) + " bytes memory increase when running : Job4");
}
if(enableLogStash) {
talendJobLog.addJobEndMessage(startTime, end, status);
try {
talendJobLogProcess(globalMap);
} catch (java.lang.Exception e) {
e.printStackTrace();
}
}
int returnCode = 0;
if(errorCode == null) {
returnCode = status != null && status.equals("failure") ? 1 : 0;
} else {
returnCode = errorCode.intValue();
}
resumeUtil.addLog("JOB_ENDED", "JOB:" + jobName, parent_part_launcher, Thread.currentThread().getId() + "", "","" + returnCode,"","","");
return returnCode;
}
// only for OSGi env
public void destroy() {
}
private java.util.Map<String, Object> getSharedConnections4REST() {
java.util.Map<String, Object> connections = new java.util.HashMap<String, Object>();
return connections;
}
private void evalParam(String arg) {
if (arg.startsWith("--resuming_logs_dir_path")) {
resuming_logs_dir_path = arg.substring(25);
} else if (arg.startsWith("--resuming_checkpoint_path")) {
resuming_checkpoint_path = arg.substring(27);
} else if (arg.startsWith("--parent_part_launcher")) {
parent_part_launcher = arg.substring(23);
} else if (arg.startsWith("--watch")) {
watch = true;
} else if (arg.startsWith("--stat_port=")) {
String portStatsStr = arg.substring(12);
if (portStatsStr != null && !portStatsStr.equals("null")) {
portStats = Integer.parseInt(portStatsStr);
}
} else if (arg.startsWith("--trace_port=")) {
portTraces = Integer.parseInt(arg.substring(13));
} else if (arg.startsWith("--client_host=")) {
clientHost = arg.substring(14);
} else if (arg.startsWith("--context=")) {
contextStr = arg.substring(10);
isDefaultContext = false;
} else if (arg.startsWith("--father_pid=")) {
fatherPid = arg.substring(13);
} else if (arg.startsWith("--root_pid=")) {
rootPid = arg.substring(11);
} else if (arg.startsWith("--father_node=")) {
fatherNode = arg.substring(14);
} else if (arg.startsWith("--pid=")) {
pid = arg.substring(6);
} else if (arg.startsWith("--context_type")) {
String keyValue = arg.substring(15);
int index = -1;
if (keyValue != null && (index = keyValue.indexOf('=')) > -1) {
if (fatherPid==null) {
context_param.setContextType(keyValue.substring(0, index), replaceEscapeChars(keyValue.substring(index + 1)));
} else { // the subjob won't escape the especial chars
context_param.setContextType(keyValue.substring(0, index), keyValue.substring(index + 1) );
}
}
} else if (arg.startsWith("--context_param")) {
String keyValue = arg.substring(16);
int index = -1;
if (keyValue != null && (index = keyValue.indexOf('=')) > -1) {
if (fatherPid==null) {
context_param.put(keyValue.substring(0, index), replaceEscapeChars(keyValue.substring(index + 1)));
} else { // the subjob won't escape the especial chars
context_param.put(keyValue.substring(0, index), keyValue.substring(index + 1) );
}
}
} else if (arg.startsWith("--log4jLevel=")) {
log4jLevel = arg.substring(13);
} else if (arg.startsWith("--monitoring=")) {//for trunjob call
enableLogStash = "true".equalsIgnoreCase(arg.substring(13));
}
if(!enableLogStash) {
enableLogStash = "true".equalsIgnoreCase(System.getProperty("monitoring"));
}
}
private static final String NULL_VALUE_EXPRESSION_IN_COMMAND_STRING_FOR_CHILD_JOB_ONLY = "<TALEND_NULL>";
private final String[][] escapeChars = {
{"\\\\","\\"},{"\\n","\n"},{"\\'","\'"},{"\\r","\r"},
{"\\f","\f"},{"\\b","\b"},{"\\t","\t"}
};
private String replaceEscapeChars (String keyValue) {
if (keyValue == null || ("").equals(keyValue.trim())) {
return keyValue;
}
StringBuilder result = new StringBuilder();
int currIndex = 0;
while (currIndex < keyValue.length()) {
int index = -1;
// judege if the left string includes escape chars
for (String[] strArray : escapeChars) {
index = keyValue.indexOf(strArray[0],currIndex);
if (index>=0) {
result.append(keyValue.substring(currIndex, index + strArray[0].length()).replace(strArray[0], strArray[1]));
currIndex = index + strArray[0].length();
break;
}
}
// if the left string doesn't include escape chars, append the left into the result
if (index < 0) {
result.append(keyValue.substring(currIndex));
currIndex = currIndex + keyValue.length();
}
}
return result.toString();
}
public Integer getErrorCode() {
return errorCode;
}
public String getStatus() {
return status;
}
ResumeUtil resumeUtil = null;
}
/************************************************************************************************
* 124832 characters generated by Talend Big Data Platform
* on the 2019/10/10 11:14:17 JST
************************************************************************************************/
package local_project.job5_0_1;
import routines.DataOperation;
import routines.TalendDataGenerator;
import routines.DataQuality;
import routines.Relational;
import routines.DataQualityDependencies;
import routines.Mathematical;
import routines.SQLike;
import routines.Numeric;
import routines.TalendStringUtil;
import routines.TalendString;
import routines.DQTechnical;
import routines.StringHandling;
import routines.DataMasking;
import routines.TalendDate;
import routines.DqStringHandling;
import routines.system.*;
import routines.system.api.*;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.math.BigDecimal;
import java.io.ByteArrayOutputStream;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.ObjectOutputStream;
import java.io.ObjectInputStream;
import java.io.IOException;
import java.util.Comparator;
@SuppressWarnings("unused")
/**
* Job: Job5 Purpose: <br>
* Description: <br>
*
* @author [email protected]
* @version 7.2.1.20190620_1446
* @status
*/
public class Job5 implements TalendJob {
static {
System.setProperty("TalendJob.log", "Job5.log");
}
private static org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(Job5.class);
protected static void logIgnoredError(String message, Throwable cause) {
log.error(message, cause);
}
public final Object obj = new Object();
// for transmiting parameters purpose
private Object valueObject = null;
public Object getValueObject() {
return this.valueObject;
}
public void setValueObject(Object valueObject) {
this.valueObject = valueObject;
}
private final static String defaultCharset = java.nio.charset.Charset.defaultCharset().name();
private final static String utf8Charset = "UTF-8";
// contains type for every context property
public class PropertiesWithType extends java.util.Properties {
private static final long serialVersionUID = 1L;
private java.util.Map<String, String> propertyTypes = new java.util.HashMap<>();
public PropertiesWithType(java.util.Properties properties) {
super(properties);
}
public PropertiesWithType() {
super();
}
public void setContextType(String key, String type) {
propertyTypes.put(key, type);
}
public String getContextType(String key) {
return propertyTypes.get(key);
}
}
// create and load default properties
private java.util.Properties defaultProps = new java.util.Properties();
// create application properties with default
public class ContextProperties extends PropertiesWithType {
private static final long serialVersionUID = 1L;
public ContextProperties(java.util.Properties properties) {
super(properties);
}
public ContextProperties() {
super();
}
public void synchronizeContext() {
if (ctxRegion != null) {
this.setProperty("ctxRegion", ctxRegion.toString());
}
}
public String ctxRegion;
public String getCtxRegion() {
return this.ctxRegion;
}
}
protected ContextProperties context = new ContextProperties(); // will be instanciated by MS.
public ContextProperties getContext() {
return this.context;
}
private final String jobVersion = "0.1";
private final String jobName = "Job5";
private final String projectName = "LOCAL_PROJECT";
public Integer errorCode = null;
private String currentComponent = "";
private final java.util.Map<String, Object> globalMap = new java.util.HashMap<String, Object>();
private final static java.util.Map<String, Object> junitGlobalMap = new java.util.HashMap<String, Object>();
private final java.util.Map<String, Long> start_Hash = new java.util.HashMap<String, Long>();
private final java.util.Map<String, Long> end_Hash = new java.util.HashMap<String, Long>();
private final java.util.Map<String, Boolean> ok_Hash = new java.util.HashMap<String, Boolean>();
public final java.util.List<String[]> globalBuffer = new java.util.ArrayList<String[]>();
private RunStat runStat = new RunStat();
// OSGi DataSource
private final static String KEY_DB_DATASOURCES = "KEY_DB_DATASOURCES";
private final static String KEY_DB_DATASOURCES_RAW = "KEY_DB_DATASOURCES_RAW";
public void setDataSources(java.util.Map<String, javax.sql.DataSource> dataSources) {
java.util.Map<String, routines.system.TalendDataSource> talendDataSources = new java.util.HashMap<String, routines.system.TalendDataSource>();
for (java.util.Map.Entry<String, javax.sql.DataSource> dataSourceEntry : dataSources.entrySet()) {
talendDataSources.put(dataSourceEntry.getKey(),
new routines.system.TalendDataSource(dataSourceEntry.getValue()));
}
globalMap.put(KEY_DB_DATASOURCES, talendDataSources);
globalMap.put(KEY_DB_DATASOURCES_RAW, new java.util.HashMap<String, javax.sql.DataSource>(dataSources));
}
JobStructureCatcherUtils talendJobLog = new JobStructureCatcherUtils(jobName, "_n-n8wOmdEemaRIEX-Ts-VA", "0.1");
org.talend.job.audit.JobAuditLogger auditLogger_talendJobLog = null;
private final java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream();
private final java.io.PrintStream errorMessagePS = new java.io.PrintStream(new java.io.BufferedOutputStream(baos));
public String getExceptionStackTrace() {
if ("failure".equals(this.getStatus())) {
errorMessagePS.flush();
return baos.toString();
}
return null;
}
private Exception exception;
public Exception getException() {
if ("failure".equals(this.getStatus())) {
return this.exception;
}
return null;
}
private class TalendException extends Exception {
private static final long serialVersionUID = 1L;
private java.util.Map<String, Object> globalMap = null;
private Exception e = null;
private String currentComponent = null;
private String virtualComponentName = null;
public void setVirtualComponentName(String virtualComponentName) {
this.virtualComponentName = virtualComponentName;
}
private TalendException(Exception e, String errorComponent, final java.util.Map<String, Object> globalMap) {
this.currentComponent = errorComponent;
this.globalMap = globalMap;
this.e = e;
}
public Exception getException() {
return this.e;
}
public String getCurrentComponent() {
return this.currentComponent;
}
public String getExceptionCauseMessage(Exception e) {
Throwable cause = e;
String message = null;
int i = 10;
while (null != cause && 0 < i--) {
message = cause.getMessage();
if (null == message) {
cause = cause.getCause();
} else {
break;
}
}
if (null == message) {
message = e.getClass().getName();
}
return message;
}
@Override
public void printStackTrace() {
if (!(e instanceof TalendException || e instanceof TDieException)) {
if (virtualComponentName != null && currentComponent.indexOf(virtualComponentName + "_") == 0) {
globalMap.put(virtualComponentName + "_ERROR_MESSAGE", getExceptionCauseMessage(e));
}
globalMap.put(currentComponent + "_ERROR_MESSAGE", getExceptionCauseMessage(e));
System.err.println("Exception in component " + currentComponent + " (" + jobName + ")");
}
if (!(e instanceof TDieException)) {
if (e instanceof TalendException) {
e.printStackTrace();
} else {
e.printStackTrace();
e.printStackTrace(errorMessagePS);
Job5.this.exception = e;
}
}
if (!(e instanceof TalendException)) {
try {
for (java.lang.reflect.Method m : this.getClass().getEnclosingClass().getMethods()) {
if (m.getName().compareTo(currentComponent + "_error") == 0) {
m.invoke(Job5.this, new Object[] { e, currentComponent, globalMap });
break;
}
}
if (!(e instanceof TDieException)) {
}
} catch (Exception e) {
this.e.printStackTrace();
}
}
}
}
public void tPrejob_1_error(Exception exception, String errorComponent,
final java.util.Map<String, Object> globalMap) throws TalendException {
end_Hash.put(errorComponent, System.currentTimeMillis());
status = "failure";
tPrejob_1_onSubJobError(exception, errorComponent, globalMap);
}
public void tFileDelete_1_error(Exception exception, String errorComponent,
final java.util.Map<String, Object> globalMap) throws TalendException {
end_Hash.put(errorComponent, System.currentTimeMillis());
status = "failure";
tFileDelete_1_onSubJobError(exception, errorComponent, globalMap);
}
public void tRunJob_1_error(Exception exception, String errorComponent,
final java.util.Map<String, Object> globalMap) throws TalendException {
end_Hash.put(errorComponent, System.currentTimeMillis());
status = "failure";
tRunJob_1_onSubJobError(exception, errorComponent, globalMap);
}
public void talendJobLog_error(Exception exception, String errorComponent,
final java.util.Map<String, Object> globalMap) throws TalendException {
end_Hash.put(errorComponent, System.currentTimeMillis());
status = "failure";
talendJobLog_onSubJobError(exception, errorComponent, globalMap);
}
public void tPrejob_1_onSubJobError(Exception exception, String errorComponent,
final java.util.Map<String, Object> globalMap) throws TalendException {
resumeUtil.addLog("SYSTEM_LOG", "NODE:" + errorComponent, "", Thread.currentThread().getId() + "", "FATAL", "",
exception.getMessage(), ResumeUtil.getExceptionStackTrace(exception), "");
}
public void tFileDelete_1_onSubJobError(Exception exception, String errorComponent,
final java.util.Map<String, Object> globalMap) throws TalendException {
resumeUtil.addLog("SYSTEM_LOG", "NODE:" + errorComponent, "", Thread.currentThread().getId() + "", "FATAL", "",
exception.getMessage(), ResumeUtil.getExceptionStackTrace(exception), "");
}
public void tRunJob_1_onSubJobError(Exception exception, String errorComponent,
final java.util.Map<String, Object> globalMap) throws TalendException {
resumeUtil.addLog("SYSTEM_LOG", "NODE:" + errorComponent, "", Thread.currentThread().getId() + "", "FATAL", "",
exception.getMessage(), ResumeUtil.getExceptionStackTrace(exception), "");
}
public void talendJobLog_onSubJobError(Exception exception, String errorComponent,
final java.util.Map<String, Object> globalMap) throws TalendException {
resumeUtil.addLog("SYSTEM_LOG", "NODE:" + errorComponent, "", Thread.currentThread().getId() + "", "FATAL", "",
exception.getMessage(), ResumeUtil.getExceptionStackTrace(exception), "");
}
public void tPrejob_1Process(final java.util.Map<String, Object> globalMap) throws TalendException {
globalMap.put("tPrejob_1_SUBPROCESS_STATE", 0);
final boolean execStat = this.execStat;
String iterateId = "";
String currentComponent = "";
java.util.Map<String, Object> resourceMap = new java.util.HashMap<String, Object>();
try {
// TDI-39566 avoid throwing an useless Exception
boolean resumeIt = true;
if (globalResumeTicket == false && resumeEntryMethodName != null) {
String currentMethodName = new java.lang.Exception().getStackTrace()[0].getMethodName();
resumeIt = resumeEntryMethodName.equals(currentMethodName);
}
if (resumeIt || globalResumeTicket) { // start the resume
globalResumeTicket = true;
/**
* [tPrejob_1 begin ] start
*/
ok_Hash.put("tPrejob_1", false);
start_Hash.put("tPrejob_1", System.currentTimeMillis());
currentComponent = "tPrejob_1";
int tos_count_tPrejob_1 = 0;
if (enableLogStash) {
talendJobLog.addComponentMessage("tPrejob_1", "tPrejob");
talendJobLogProcess(globalMap);
}
/**
* [tPrejob_1 begin ] stop
*/
/**
* [tPrejob_1 main ] start
*/
currentComponent = "tPrejob_1";
tos_count_tPrejob_1++;
/**
* [tPrejob_1 main ] stop
*/
/**
* [tPrejob_1 process_data_begin ] start
*/
currentComponent = "tPrejob_1";
/**
* [tPrejob_1 process_data_begin ] stop
*/
/**
* [tPrejob_1 process_data_end ] start
*/
currentComponent = "tPrejob_1";
/**
* [tPrejob_1 process_data_end ] stop
*/
/**
* [tPrejob_1 end ] start
*/
currentComponent = "tPrejob_1";
ok_Hash.put("tPrejob_1", true);
end_Hash.put("tPrejob_1", System.currentTimeMillis());
tFileDelete_1Process(globalMap);
/**
* [tPrejob_1 end ] stop
*/
} // end the resume
} catch (java.lang.Exception e) {
if (!(e instanceof TalendException)) {
log.fatal(currentComponent + " " + e.getMessage(), e);
}
TalendException te = new TalendException(e, currentComponent, globalMap);
throw te;
} catch (java.lang.Error error) {
throw error;
} finally {
try {
/**
* [tPrejob_1 finally ] start
*/
currentComponent = "tPrejob_1";
/**
* [tPrejob_1 finally ] stop
*/
} catch (java.lang.Exception e) {
// ignore
} catch (java.lang.Error error) {
// ignore
}
resourceMap = null;
}
globalMap.put("tPrejob_1_SUBPROCESS_STATE", 1);
}
public void tFileDelete_1Process(final java.util.Map<String, Object> globalMap) throws TalendException {
globalMap.put("tFileDelete_1_SUBPROCESS_STATE", 0);
final boolean execStat = this.execStat;
String iterateId = "";
String currentComponent = "";
java.util.Map<String, Object> resourceMap = new java.util.HashMap<String, Object>();
try {
// TDI-39566 avoid throwing an useless Exception
boolean resumeIt = true;
if (globalResumeTicket == false && resumeEntryMethodName != null) {
String currentMethodName = new java.lang.Exception().getStackTrace()[0].getMethodName();
resumeIt = resumeEntryMethodName.equals(currentMethodName);
}
if (resumeIt || globalResumeTicket) { // start the resume
globalResumeTicket = true;
/**
* [tFileDelete_1 begin ] start
*/
ok_Hash.put("tFileDelete_1", false);
start_Hash.put("tFileDelete_1", System.currentTimeMillis());
currentComponent = "tFileDelete_1";
int tos_count_tFileDelete_1 = 0;
if (log.isDebugEnabled())
log.debug("tFileDelete_1 - " + ("Start to work."));
if (log.isDebugEnabled()) {
class BytesLimit65535_tFileDelete_1 {
public void limitLog4jByte() throws Exception {
StringBuilder log4jParamters_tFileDelete_1 = new StringBuilder();
log4jParamters_tFileDelete_1.append("Parameters:");
log4jParamters_tFileDelete_1.append("FILENAME" + " = "
+ "\"C:/Users/hryk.hoshi/Downloads/out_\"+context.ctxRegion+\".csv\"");
log4jParamters_tFileDelete_1.append(" | ");
log4jParamters_tFileDelete_1.append("FAILON" + " = " + "true");
log4jParamters_tFileDelete_1.append(" | ");
log4jParamters_tFileDelete_1.append("FOLDER" + " = " + "false");
log4jParamters_tFileDelete_1.append(" | ");
log4jParamters_tFileDelete_1.append("FOLDER_FILE" + " = " + "false");
log4jParamters_tFileDelete_1.append(" | ");
if (log.isDebugEnabled())
log.debug("tFileDelete_1 - " + (log4jParamters_tFileDelete_1));
}
}
new BytesLimit65535_tFileDelete_1().limitLog4jByte();
}
if (enableLogStash) {
talendJobLog.addComponentMessage("tFileDelete_1", "tFileDelete");
talendJobLogProcess(globalMap);
}
/**
* [tFileDelete_1 begin ] stop
*/
/**
* [tFileDelete_1 main ] start
*/
currentComponent = "tFileDelete_1";
final StringBuffer log4jSb_tFileDelete_1 = new StringBuffer();
class DeleteFoldertFileDelete_1 {
/**
* delete all the sub-files in 'file'
*
* @param file
*/
public boolean delete(java.io.File file) {
java.io.File[] files = file.listFiles();
for (int i = 0; i < files.length; i++) {
if (files[i].isFile()) {
files[i].delete();
} else if (files[i].isDirectory()) {
if (!files[i].delete()) {
delete(files[i]);
}
}
}
deleteDirectory(file);
return file.delete();
}
/**
* delete all the sub-folders in 'file'
*
* @param file
*/
private void deleteDirectory(java.io.File file) {
java.io.File[] filed = file.listFiles();
for (int i = 0; i < filed.length; i++) {
if (filed[i].isDirectory()) {
deleteDirectory(filed[i]);
}
filed[i].delete();
}
}
}
java.io.File file_tFileDelete_1 = new java.io.File(
"C:/Users/hryk.hoshi/Downloads/out_" + context.ctxRegion + ".csv");
if (file_tFileDelete_1.exists() && file_tFileDelete_1.isFile()) {
if (file_tFileDelete_1.delete()) {
globalMap.put("tFileDelete_1_CURRENT_STATUS", "File deleted.");
log.info("tFileDelete_1 - File : " + file_tFileDelete_1.getAbsolutePath() + " is deleted.");
} else {
globalMap.put("tFileDelete_1_CURRENT_STATUS", "No file deleted.");
throw new RuntimeException(
"File " + file_tFileDelete_1.getAbsolutePath() + " can not be deleted.");
}
} else {
globalMap.put("tFileDelete_1_CURRENT_STATUS", "File does not exist or is invalid.");
throw new RuntimeException("File " + file_tFileDelete_1.getAbsolutePath()
+ " does not exist or is invalid or is not a file.");
}
globalMap.put("tFileDelete_1_DELETE_PATH",
"C:/Users/hryk.hoshi/Downloads/out_" + context.ctxRegion + ".csv");
tos_count_tFileDelete_1++;
/**
* [tFileDelete_1 main ] stop
*/
/**
* [tFileDelete_1 process_data_begin ] start
*/
currentComponent = "tFileDelete_1";
/**
* [tFileDelete_1 process_data_begin ] stop
*/
/**
* [tFileDelete_1 process_data_end ] start
*/
currentComponent = "tFileDelete_1";
/**
* [tFileDelete_1 process_data_end ] stop
*/
/**
* [tFileDelete_1 end ] start
*/
currentComponent = "tFileDelete_1";
if (log.isDebugEnabled())
log.debug("tFileDelete_1 - " + ("Done."));
ok_Hash.put("tFileDelete_1", true);
end_Hash.put("tFileDelete_1", System.currentTimeMillis());
/**
* [tFileDelete_1 end ] stop
*/
} // end the resume
} catch (java.lang.Exception e) {
if (!(e instanceof TalendException)) {
log.fatal(currentComponent + " " + e.getMessage(), e);
}
TalendException te = new TalendException(e, currentComponent, globalMap);
throw te;
} catch (java.lang.Error error) {
throw error;
} finally {
try {
/**
* [tFileDelete_1 finally ] start
*/
currentComponent = "tFileDelete_1";
/**
* [tFileDelete_1 finally ] stop
*/
} catch (java.lang.Exception e) {
// ignore
} catch (java.lang.Error error) {
// ignore
}
resourceMap = null;
}
globalMap.put("tFileDelete_1_SUBPROCESS_STATE", 1);
}
public void tRunJob_1Process(final java.util.Map<String, Object> globalMap) throws TalendException {
globalMap.put("tRunJob_1_SUBPROCESS_STATE", 0);
final boolean execStat = this.execStat;
String iterateId = "";
String currentComponent = "";
java.util.Map<String, Object> resourceMap = new java.util.HashMap<String, Object>();
try {
// TDI-39566 avoid throwing an useless Exception
boolean resumeIt = true;
if (globalResumeTicket == false && resumeEntryMethodName != null) {
String currentMethodName = new java.lang.Exception().getStackTrace()[0].getMethodName();
resumeIt = resumeEntryMethodName.equals(currentMethodName);
}
if (resumeIt || globalResumeTicket) { // start the resume
globalResumeTicket = true;
/**
* [tRunJob_1 begin ] start
*/
ok_Hash.put("tRunJob_1", false);
start_Hash.put("tRunJob_1", System.currentTimeMillis());
currentComponent = "tRunJob_1";
int tos_count_tRunJob_1 = 0;
if (log.isDebugEnabled())
log.debug("tRunJob_1 - " + ("Start to work."));
if (log.isDebugEnabled()) {
class BytesLimit65535_tRunJob_1 {
public void limitLog4jByte() throws Exception {
StringBuilder log4jParamters_tRunJob_1 = new StringBuilder();
log4jParamters_tRunJob_1.append("Parameters:");
log4jParamters_tRunJob_1.append("USE_DYNAMIC_JOB" + " = " + "false");
log4jParamters_tRunJob_1.append(" | ");
log4jParamters_tRunJob_1.append("PROCESS" + " = " + "Job4");
log4jParamters_tRunJob_1.append(" | ");
log4jParamters_tRunJob_1.append("USE_INDEPENDENT_PROCESS" + " = " + "false");
log4jParamters_tRunJob_1.append(" | ");
log4jParamters_tRunJob_1.append("DIE_ON_CHILD_ERROR" + " = " + "true");
log4jParamters_tRunJob_1.append(" | ");
log4jParamters_tRunJob_1.append("TRANSMIT_WHOLE_CONTEXT" + " = " + "false");
log4jParamters_tRunJob_1.append(" | ");
log4jParamters_tRunJob_1.append("CONTEXTPARAMS" + " = " + "[]");
log4jParamters_tRunJob_1.append(" | ");
log4jParamters_tRunJob_1.append("PROPAGATE_CHILD_RESULT" + " = " + "false");
log4jParamters_tRunJob_1.append(" | ");
log4jParamters_tRunJob_1.append("PRINT_PARAMETER" + " = " + "false");
log4jParamters_tRunJob_1.append(" | ");
if (log.isDebugEnabled())
log.debug("tRunJob_1 - " + (log4jParamters_tRunJob_1));
}
}
new BytesLimit65535_tRunJob_1().limitLog4jByte();
}
if (enableLogStash) {
talendJobLog.addComponentMessage("tRunJob_1", "tRunJob");
talendJobLogProcess(globalMap);
}
/**
* [tRunJob_1 begin ] stop
*/
/**
* [tRunJob_1 main ] start
*/
currentComponent = "tRunJob_1";
java.util.List<String> paraList_tRunJob_1 = new java.util.ArrayList<String>();
paraList_tRunJob_1.add("--father_pid=" + pid);
paraList_tRunJob_1.add("--root_pid=" + rootPid);
paraList_tRunJob_1.add("--father_node=tRunJob_1");
paraList_tRunJob_1.add("--context=Default");
if (!"".equals(log4jLevel)) {
paraList_tRunJob_1.add("--log4jLevel=" + log4jLevel);
}
if (enableLogStash) {
paraList_tRunJob_1.add("--monitoring=" + enableLogStash);
}
// for feature:10589
paraList_tRunJob_1.add("--stat_port=" + portStats);
if (resuming_logs_dir_path != null) {
paraList_tRunJob_1.add("--resuming_logs_dir_path=" + resuming_logs_dir_path);
}
String childResumePath_tRunJob_1 = ResumeUtil.getChildJobCheckPointPath(resuming_checkpoint_path);
String tRunJobName_tRunJob_1 = ResumeUtil.getRighttRunJob(resuming_checkpoint_path);
if ("tRunJob_1".equals(tRunJobName_tRunJob_1) && childResumePath_tRunJob_1 != null) {
paraList_tRunJob_1.add("--resuming_checkpoint_path="
+ ResumeUtil.getChildJobCheckPointPath(resuming_checkpoint_path));
}
paraList_tRunJob_1.add("--parent_part_launcher=JOB:" + jobName + "/NODE:tRunJob_1");
java.util.Map<String, Object> parentContextMap_tRunJob_1 = new java.util.HashMap<String, Object>();
Object obj_tRunJob_1 = null;
local_project.job4_0_1.Job4 childJob_tRunJob_1 = new local_project.job4_0_1.Job4();
// pass DataSources
java.util.Map<String, routines.system.TalendDataSource> talendDataSources_tRunJob_1 = (java.util.Map<String, routines.system.TalendDataSource>) globalMap
.get(KEY_DB_DATASOURCES);
if (null != talendDataSources_tRunJob_1) {
java.util.Map<String, javax.sql.DataSource> dataSources_tRunJob_1 = new java.util.HashMap<String, javax.sql.DataSource>();
for (java.util.Map.Entry<String, routines.system.TalendDataSource> talendDataSourceEntry_tRunJob_1 : talendDataSources_tRunJob_1
.entrySet()) {
dataSources_tRunJob_1.put(talendDataSourceEntry_tRunJob_1.getKey(),
talendDataSourceEntry_tRunJob_1.getValue().getRawDataSource());
}
childJob_tRunJob_1.setDataSources(dataSources_tRunJob_1);
}
childJob_tRunJob_1.parentContextMap = parentContextMap_tRunJob_1;
log.info(
"tRunJob_1 - The child job 'local_project.job4_0_1.Job4' starts on the version '0.1' with the context 'Default'.");
String[][] childReturn_tRunJob_1 = childJob_tRunJob_1
.runJob((String[]) paraList_tRunJob_1.toArray(new String[paraList_tRunJob_1.size()]));
log.info("tRunJob_1 - The child job 'local_project.job4_0_1.Job4' is done.");
errorCode = childJob_tRunJob_1.getErrorCode();
if (childJob_tRunJob_1.getErrorCode() == null) {
globalMap.put("tRunJob_1_CHILD_RETURN_CODE",
childJob_tRunJob_1.getStatus() != null && ("failure").equals(childJob_tRunJob_1.getStatus())
? 1
: 0);
} else {
globalMap.put("tRunJob_1_CHILD_RETURN_CODE", childJob_tRunJob_1.getErrorCode());
}
if (childJob_tRunJob_1.getExceptionStackTrace() != null) {
globalMap.put("tRunJob_1_CHILD_EXCEPTION_STACKTRACE", childJob_tRunJob_1.getExceptionStackTrace());
}
if (childJob_tRunJob_1.getErrorCode() != null || ("failure").equals(childJob_tRunJob_1.getStatus())) {
java.lang.Exception ce_tRunJob_1 = childJob_tRunJob_1.getException();
throw new RuntimeException("Child job running failed.\n" + ((ce_tRunJob_1 != null)
? (ce_tRunJob_1.getClass().getName() + ": " + ce_tRunJob_1.getMessage())
: ""));
}
tos_count_tRunJob_1++;
/**
* [tRunJob_1 main ] stop
*/
/**
* [tRunJob_1 process_data_begin ] start
*/
currentComponent = "tRunJob_1";
/**
* [tRunJob_1 process_data_begin ] stop
*/
/**
* [tRunJob_1 process_data_end ] start
*/
currentComponent = "tRunJob_1";
/**
* [tRunJob_1 process_data_end ] stop
*/
/**
* [tRunJob_1 end ] start
*/
currentComponent = "tRunJob_1";
if (log.isDebugEnabled())
log.debug("tRunJob_1 - " + ("Done."));
ok_Hash.put("tRunJob_1", true);
end_Hash.put("tRunJob_1", System.currentTimeMillis());
/**
* [tRunJob_1 end ] stop
*/
} // end the resume
} catch (java.lang.Exception e) {
if (!(e instanceof TalendException)) {
log.fatal(currentComponent + " " + e.getMessage(), e);
}
TalendException te = new TalendException(e, currentComponent, globalMap);
throw te;
} catch (java.lang.Error error) {
throw error;
} finally {
try {
/**
* [tRunJob_1 finally ] start
*/
currentComponent = "tRunJob_1";
/**
* [tRunJob_1 finally ] stop
*/
} catch (java.lang.Exception e) {
// ignore
} catch (java.lang.Error error) {
// ignore
}
resourceMap = null;
}
globalMap.put("tRunJob_1_SUBPROCESS_STATE", 1);
}
public void talendJobLogProcess(final java.util.Map<String, Object> globalMap) throws TalendException {
globalMap.put("talendJobLog_SUBPROCESS_STATE", 0);
final boolean execStat = this.execStat;
String iterateId = "";
String currentComponent = "";
java.util.Map<String, Object> resourceMap = new java.util.HashMap<String, Object>();
try {
// TDI-39566 avoid throwing an useless Exception
boolean resumeIt = true;
if (globalResumeTicket == false && resumeEntryMethodName != null) {
String currentMethodName = new java.lang.Exception().getStackTrace()[0].getMethodName();
resumeIt = resumeEntryMethodName.equals(currentMethodName);
}
if (resumeIt || globalResumeTicket) { // start the resume
globalResumeTicket = true;
/**
* [talendJobLog begin ] start
*/
ok_Hash.put("talendJobLog", false);
start_Hash.put("talendJobLog", System.currentTimeMillis());
currentComponent = "talendJobLog";
int tos_count_talendJobLog = 0;
for (JobStructureCatcherUtils.JobStructureCatcherMessage jcm : talendJobLog.getMessages()) {
org.talend.logging.audit.Context log_context_talendJobLog = null;
if (jcm.component_name == null) {// job level log
if (jcm.status == null) {// job start
log_context_talendJobLog = org.talend.job.audit.JobContextBuilder.create()
.jobName(jcm.job_name).jobId(jcm.job_id).jobVersion(jcm.job_version)
.timestamp(jcm.moment).build();
auditLogger_talendJobLog.jobstart(log_context_talendJobLog);
} else {// job end
long timeMS = jcm.end_time - jcm.start_time;
String duration = String.format(java.util.Locale.US, "%1$.2fs", (timeMS * 1.0) / 1000);
log_context_talendJobLog = org.talend.job.audit.JobContextBuilder.create()
.jobName(jcm.job_name).jobId(jcm.job_id).jobVersion(jcm.job_version)
.timestamp(jcm.moment).duration(duration).status(jcm.status).build();
auditLogger_talendJobLog.jobstop(log_context_talendJobLog);
}
} else if (jcm.current_connector == null) {// component log
log_context_talendJobLog = org.talend.job.audit.JobContextBuilder.create().jobName(jcm.job_name)
.jobId(jcm.job_id).jobVersion(jcm.job_version).connectorType(jcm.component_name)
.connectorId(jcm.component_id).build();
auditLogger_talendJobLog.runcomponent(log_context_talendJobLog);
} else {// component connector meter log
long timeMS = jcm.end_time - jcm.start_time;
String duration = String.format(java.util.Locale.US, "%1$.2fs", (timeMS * 1.0) / 1000);
if (jcm.current_connector_as_input) {// log current component input line
log_context_talendJobLog = org.talend.job.audit.JobContextBuilder.create()
.jobName(jcm.job_name).jobId(jcm.job_id).jobVersion(jcm.job_version)
.connectorType(jcm.component_name).connectorId(jcm.component_id)
.connectionName(jcm.current_connector).connectionType(jcm.current_connector_type)
.rows(jcm.total_row_number).duration(duration).build();
auditLogger_talendJobLog.flowInput(log_context_talendJobLog);
} else {// log current component output/reject line
log_context_talendJobLog = org.talend.job.audit.JobContextBuilder.create()
.jobName(jcm.job_name).jobId(jcm.job_id).jobVersion(jcm.job_version)
.connectorType(jcm.component_name).connectorId(jcm.component_id)
.connectionName(jcm.current_connector).connectionType(jcm.current_connector_type)
.rows(jcm.total_row_number).duration(duration).build();
auditLogger_talendJobLog.flowOutput(log_context_talendJobLog);
}
}
}
/**
* [talendJobLog begin ] stop
*/
/**
* [talendJobLog main ] start
*/
currentComponent = "talendJobLog";
tos_count_talendJobLog++;
/**
* [talendJobLog main ] stop
*/
/**
* [talendJobLog process_data_begin ] start
*/
currentComponent = "talendJobLog";
/**
* [talendJobLog process_data_begin ] stop
*/
/**
* [talendJobLog process_data_end ] start
*/
currentComponent = "talendJobLog";
/**
* [talendJobLog process_data_end ] stop
*/
/**
* [talendJobLog end ] start
*/
currentComponent = "talendJobLog";
ok_Hash.put("talendJobLog", true);
end_Hash.put("talendJobLog", System.currentTimeMillis());
/**
* [talendJobLog end ] stop
*/
} // end the resume
} catch (java.lang.Exception e) {
if (!(e instanceof TalendException)) {
log.fatal(currentComponent + " " + e.getMessage(), e);
}
TalendException te = new TalendException(e, currentComponent, globalMap);
throw te;
} catch (java.lang.Error error) {
throw error;
} finally {
try {
/**
* [talendJobLog finally ] start
*/
currentComponent = "talendJobLog";
/**
* [talendJobLog finally ] stop
*/
} catch (java.lang.Exception e) {
// ignore
} catch (java.lang.Error error) {
// ignore
}
resourceMap = null;
}
globalMap.put("talendJobLog_SUBPROCESS_STATE", 1);
}
public String resuming_logs_dir_path = null;
public String resuming_checkpoint_path = null;
public String parent_part_launcher = null;
private String resumeEntryMethodName = null;
private boolean globalResumeTicket = false;
public boolean watch = false;
// portStats is null, it means don't execute the statistics
public Integer portStats = null;
public int portTraces = 4334;
public String clientHost;
public String defaultClientHost = "localhost";
public String contextStr = "Default";
public boolean isDefaultContext = true;
public String pid = "0";
public String rootPid = null;
public String fatherPid = null;
public String fatherNode = null;
public long startTime = 0;
public boolean isChildJob = false;
public String log4jLevel = "";
private boolean enableLogStash;
private boolean execStat = true;
private ThreadLocal<java.util.Map<String, String>> threadLocal = new ThreadLocal<java.util.Map<String, String>>() {
protected java.util.Map<String, String> initialValue() {
java.util.Map<String, String> threadRunResultMap = new java.util.HashMap<String, String>();
threadRunResultMap.put("errorCode", null);
threadRunResultMap.put("status", "");
return threadRunResultMap;
};
};
private PropertiesWithType context_param = new PropertiesWithType();
public java.util.Map<String, Object> parentContextMap = new java.util.HashMap<String, Object>();
public String status = "";
public static void main(String[] args) {
final Job5 Job5Class = new Job5();
int exitCode = Job5Class.runJobInTOS(args);
if (exitCode == 0) {
log.info("TalendJob: 'Job5' - Done.");
}
System.exit(exitCode);
}
public String[][] runJob(String[] args) {
int exitCode = runJobInTOS(args);
String[][] bufferValue = new String[][] { { Integer.toString(exitCode) } };
return bufferValue;
}
public boolean hastBufferOutputComponent() {
boolean hastBufferOutput = false;
return hastBufferOutput;
}
public int runJobInTOS(String[] args) {
// reset status
status = "";
String lastStr = "";
for (String arg : args) {
if (arg.equalsIgnoreCase("--context_param")) {
lastStr = arg;
} else if (lastStr.equals("")) {
evalParam(arg);
} else {
evalParam(lastStr + " " + arg);
lastStr = "";
}
}
if (!"".equals(log4jLevel)) {
if ("trace".equalsIgnoreCase(log4jLevel)) {
log.setLevel(org.apache.log4j.Level.TRACE);
} else if ("debug".equalsIgnoreCase(log4jLevel)) {
log.setLevel(org.apache.log4j.Level.DEBUG);
} else if ("info".equalsIgnoreCase(log4jLevel)) {
log.setLevel(org.apache.log4j.Level.INFO);
} else if ("warn".equalsIgnoreCase(log4jLevel)) {
log.setLevel(org.apache.log4j.Level.WARN);
} else if ("error".equalsIgnoreCase(log4jLevel)) {
log.setLevel(org.apache.log4j.Level.ERROR);
} else if ("fatal".equalsIgnoreCase(log4jLevel)) {
log.setLevel(org.apache.log4j.Level.FATAL);
} else if ("off".equalsIgnoreCase(log4jLevel)) {
log.setLevel(org.apache.log4j.Level.OFF);
}
org.apache.log4j.Logger.getRootLogger().setLevel(log.getLevel());
}
log.info("TalendJob: 'Job5' - Start.");
if (enableLogStash) {
java.util.Properties properties_talendJobLog = new java.util.Properties();
properties_talendJobLog.setProperty("root.logger", "audit");
properties_talendJobLog.setProperty("encoding", "UTF-8");
properties_talendJobLog.setProperty("application.name", "Talend Studio");
properties_talendJobLog.setProperty("service.name", "Talend Studio Job");
properties_talendJobLog.setProperty("instance.name", "Talend Studio Job Instance");
properties_talendJobLog.setProperty("propagate.appender.exceptions", "none");
properties_talendJobLog.setProperty("log.appender", "file");
properties_talendJobLog.setProperty("appender.file.path", "audit.json");
properties_talendJobLog.setProperty("appender.file.maxsize", "52428800");
properties_talendJobLog.setProperty("appender.file.maxbackup", "20");
properties_talendJobLog.setProperty("host", "false");
auditLogger_talendJobLog = org.talend.job.audit.JobEventAuditLoggerFactory
.createJobAuditLogger(properties_talendJobLog);
}
if (clientHost == null) {
clientHost = defaultClientHost;
}
if (pid == null || "0".equals(pid)) {
pid = TalendString.getAsciiRandomString(6);
}
if (rootPid == null) {
rootPid = pid;
}
if (fatherPid == null) {
fatherPid = pid;
} else {
isChildJob = true;
}
try {
// call job/subjob with an existing context, like: --context=production. if
// without this parameter, there will use the default context instead.
java.io.InputStream inContext = Job5.class.getClassLoader()
.getResourceAsStream("local_project/job5_0_1/contexts/" + contextStr + ".properties");
if (inContext == null) {
inContext = Job5.class.getClassLoader()
.getResourceAsStream("config/contexts/" + contextStr + ".properties");
}
if (inContext != null) {
// defaultProps is in order to keep the original context value
if (context != null && context.isEmpty()) {
defaultProps.load(inContext);
context = new ContextProperties(defaultProps);
}
inContext.close();
} else if (!isDefaultContext) {
// print info and job continue to run, for case: context_param is not empty.
System.err.println("Could not find the context " + contextStr);
}
if (!context_param.isEmpty()) {
context.putAll(context_param);
// set types for params from parentJobs
for (Object key : context_param.keySet()) {
String context_key = key.toString();
String context_type = context_param.getContextType(context_key);
context.setContextType(context_key, context_type);
}
}
class ContextProcessing {
private void processContext_0() {
context.setContextType("ctxRegion", "id_String");
context.ctxRegion = (String) context.getProperty("ctxRegion");
}
public void processAllContext() {
processContext_0();
}
}
new ContextProcessing().processAllContext();
} catch (java.io.IOException ie) {
System.err.println("Could not load context " + contextStr);
ie.printStackTrace();
}
// get context value from parent directly
if (parentContextMap != null && !parentContextMap.isEmpty()) {
if (parentContextMap.containsKey("ctxRegion")) {
context.ctxRegion = (String) parentContextMap.get("ctxRegion");
}
}
// Resume: init the resumeUtil
resumeEntryMethodName = ResumeUtil.getResumeEntryMethodName(resuming_checkpoint_path);
resumeUtil = new ResumeUtil(resuming_logs_dir_path, isChildJob, rootPid);
resumeUtil.initCommonInfo(pid, rootPid, fatherPid, projectName, jobName, contextStr, jobVersion);
List<String> parametersToEncrypt = new java.util.ArrayList<String>();
// Resume: jobStart
resumeUtil.addLog("JOB_STARTED", "JOB:" + jobName, parent_part_launcher, Thread.currentThread().getId() + "",
"", "", "", "", resumeUtil.convertToJsonText(context, parametersToEncrypt));
java.util.concurrent.ConcurrentHashMap<Object, Object> concurrentHashMap = new java.util.concurrent.ConcurrentHashMap<Object, Object>();
globalMap.put("concurrentHashMap", concurrentHashMap);
long startUsedMemory = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
long endUsedMemory = 0;
long end = 0;
startTime = System.currentTimeMillis();
this.globalResumeTicket = true;// to run tPreJob
try {
errorCode = null;
tPrejob_1Process(globalMap);
if (!"failure".equals(status)) {
status = "end";
}
} catch (TalendException e_tPrejob_1) {
globalMap.put("tPrejob_1_SUBPROCESS_STATE", -1);
e_tPrejob_1.printStackTrace();
}
if (enableLogStash) {
talendJobLog.addJobStartMessage();
try {
talendJobLogProcess(globalMap);
} catch (java.lang.Exception e) {
e.printStackTrace();
}
}
this.globalResumeTicket = false;// to run others jobs
try {
errorCode = null;
tRunJob_1Process(globalMap);
if (!"failure".equals(status)) {
status = "end";
}
} catch (TalendException e_tRunJob_1) {
globalMap.put("tRunJob_1_SUBPROCESS_STATE", -1);
e_tRunJob_1.printStackTrace();
}
this.globalResumeTicket = true;// to run tPostJob
end = System.currentTimeMillis();
if (watch) {
System.out.println((end - startTime) + " milliseconds");
}
endUsedMemory = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
if (false) {
System.out.println((endUsedMemory - startUsedMemory) + " bytes memory increase when running : Job5");
}
if (enableLogStash) {
talendJobLog.addJobEndMessage(startTime, end, status);
try {
talendJobLogProcess(globalMap);
} catch (java.lang.Exception e) {
e.printStackTrace();
}
}
int returnCode = 0;
if (errorCode == null) {
returnCode = status != null && status.equals("failure") ? 1 : 0;
} else {
returnCode = errorCode.intValue();
}
resumeUtil.addLog("JOB_ENDED", "JOB:" + jobName, parent_part_launcher, Thread.currentThread().getId() + "", "",
"" + returnCode, "", "", "");
return returnCode;
}
// only for OSGi env
public void destroy() {
}
private java.util.Map<String, Object> getSharedConnections4REST() {
java.util.Map<String, Object> connections = new java.util.HashMap<String, Object>();
return connections;
}
private void evalParam(String arg) {
if (arg.startsWith("--resuming_logs_dir_path")) {
resuming_logs_dir_path = arg.substring(25);
} else if (arg.startsWith("--resuming_checkpoint_path")) {
resuming_checkpoint_path = arg.substring(27);
} else if (arg.startsWith("--parent_part_launcher")) {
parent_part_launcher = arg.substring(23);
} else if (arg.startsWith("--watch")) {
watch = true;
} else if (arg.startsWith("--stat_port=")) {
String portStatsStr = arg.substring(12);
if (portStatsStr != null && !portStatsStr.equals("null")) {
portStats = Integer.parseInt(portStatsStr);
}
} else if (arg.startsWith("--trace_port=")) {
portTraces = Integer.parseInt(arg.substring(13));
} else if (arg.startsWith("--client_host=")) {
clientHost = arg.substring(14);
} else if (arg.startsWith("--context=")) {
contextStr = arg.substring(10);
isDefaultContext = false;
} else if (arg.startsWith("--father_pid=")) {
fatherPid = arg.substring(13);
} else if (arg.startsWith("--root_pid=")) {
rootPid = arg.substring(11);
} else if (arg.startsWith("--father_node=")) {
fatherNode = arg.substring(14);
} else if (arg.startsWith("--pid=")) {
pid = arg.substring(6);
} else if (arg.startsWith("--context_type")) {
String keyValue = arg.substring(15);
int index = -1;
if (keyValue != null && (index = keyValue.indexOf('=')) > -1) {
if (fatherPid == null) {
context_param.setContextType(keyValue.substring(0, index),
replaceEscapeChars(keyValue.substring(index + 1)));
} else { // the subjob won't escape the especial chars
context_param.setContextType(keyValue.substring(0, index), keyValue.substring(index + 1));
}
}
} else if (arg.startsWith("--context_param")) {
String keyValue = arg.substring(16);
int index = -1;
if (keyValue != null && (index = keyValue.indexOf('=')) > -1) {
if (fatherPid == null) {
context_param.put(keyValue.substring(0, index), replaceEscapeChars(keyValue.substring(index + 1)));
} else { // the subjob won't escape the especial chars
context_param.put(keyValue.substring(0, index), keyValue.substring(index + 1));
}
}
} else if (arg.startsWith("--log4jLevel=")) {
log4jLevel = arg.substring(13);
} else if (arg.startsWith("--monitoring=")) {// for trunjob call
enableLogStash = "true".equalsIgnoreCase(arg.substring(13));
}
if (!enableLogStash) {
enableLogStash = "true".equalsIgnoreCase(System.getProperty("monitoring"));
}
}
private static final String NULL_VALUE_EXPRESSION_IN_COMMAND_STRING_FOR_CHILD_JOB_ONLY = "<TALEND_NULL>";
private final String[][] escapeChars = { { "\\\\", "\\" }, { "\\n", "\n" }, { "\\'", "\'" }, { "\\r", "\r" },
{ "\\f", "\f" }, { "\\b", "\b" }, { "\\t", "\t" } };
private String replaceEscapeChars(String keyValue) {
if (keyValue == null || ("").equals(keyValue.trim())) {
return keyValue;
}
StringBuilder result = new StringBuilder();
int currIndex = 0;
while (currIndex < keyValue.length()) {
int index = -1;
// judege if the left string includes escape chars
for (String[] strArray : escapeChars) {
index = keyValue.indexOf(strArray[0], currIndex);
if (index >= 0) {
result.append(keyValue.substring(currIndex, index + strArray[0].length()).replace(strArray[0],
strArray[1]));
currIndex = index + strArray[0].length();
break;
}
}
// if the left string doesn't include escape chars, append the left into the
// result
if (index < 0) {
result.append(keyValue.substring(currIndex));
currIndex = currIndex + keyValue.length();
}
}
return result.toString();
}
public Integer getErrorCode() {
return errorCode;
}
public String getStatus() {
return status;
}
ResumeUtil resumeUtil = null;
}
/************************************************************************************************
* 48755 characters generated by Talend Big Data Platform on the 2019/10/10
* 11:14:17 JST
************************************************************************************************/
Recommended Posts