/**
* Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved.
* Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
*/
/* globals app, module, __dirname */
/* jshint esversion: 6 */
var translationProvider = require('../provider/translationProvider').factory.create(),
persistenceStore = require('./persistenceStore').factory.create(),
fileImporter = require('./sampleFileImporter'),
fileTranslator = require('./sampleFileTranslator'),
fileDownloader = require('./sampleFileDownloader'),
localeMap = require('./localeMap');
/**
* Manage the translation of a job within the translation connector. <br/>
* Once the job has been created and the zip file imported, the job-manager takes over the actual translation of the zip file. <br/>
* The steps involved are: <br/>
* <ul>
* <li>Import all files into the LSP: status updated to "TRANSLATING" (% complete set at 0%)</li>
* <li>Request files are translated into the target languages</li>
* <li>Poll until complete: status updated to "DOWNLOADING" (% complete updated also added during polling - stops at 99%)</li>
* <li>Download files: status updated to "TRANSLATED" (% complete updated to 100%)</li>
* <li>If any steps error: status is updated to "FAILED"</li>
* </ul>
* @constructor
* @alias SampleJobManager
*/
var SampleJobManager = function () {
// on startup - run through and re-enable all non-TRANSLATED jobs
this.restartJobs();
};
/**
* Restart any jobs that were previously running. <br/>
* This function gets called when the translation connector starts up. Any jobs that were previously running and in a state to re-start are kicked off again. <br/>
* This is a background tasks and the jobs will run until the status is updated to "TRANSLATED" or "FAILED"
*/
SampleJobManager.prototype.restartJobs = function () {
var self = this;
// get all the the existing jobs
persistenceStore.getAllJobs().then(function (allJobConfigs) {
// get all the jobs that need to be re-started
allJobConfigs.filter(function (jobConfig) {
// If the job is not finished (i.e.: TRANSLATED), we need to re-start it
// If the job has 'FAILED' then we don't restart it, they need to re-submit the job and start again
// In order to restart, the status must be at least "IMPORTING" (i.e.: not "CREATED") so we have the zip file to work from
return ['TRANSLATED', 'FAILED', 'CREATED'].indexOf(jobConfig.status) === -1;
}).map(function (jobConfig) {
// ok, we have a running job that we need to re-start, kick it off again
console.log('restarting...:' + jobConfig.properties.id + ' from: ' + jobConfig.status);
self.translateJob(jobConfig);
});
});
};
/**
* @typedef {Object} JobConfig
* @memberof SampleJobManager
* @property {string} name Name of the connector job.
* @property {string} workflowId Identifier of the Language Service Provider workflow to use to translate this job
* @property {string} authToken Bearer token to use to connect to the Language Service Provider. Since the job-manager works in the background and restarts jobs, it needs access to this token.
* @property {('CREATED'|'IMPORTING'|'TRANSLATING'|'DOWNLOADING'|'TRANSLATED')} status The status of the job.
* @property {string} statusMessage Any message included with the status update.
* @property {number} progress Percentage progress through the translation of the job.
* @property {object} properties Language Service Provider properties.
* @property {string} properties.id Identifier of the job in the connector.
* @property {string} properties.projectId Identifier of the project in the Language Service Provider.
* @property {string} translatedZipFile Name of the zip file containing all the translations for the job.
*/
/**
* @typedef {Object} file
* @memberof SampleJobManager
* @property {string} name The name of the resource to translate from the job.json file.
* @property {string} id The reference identifier for this resource in the job.json file.
* @property {string} path The name of the file on the file system.
*/
/**
* @typedef {Object} translation
* @memberof SampleJobManager.contentType.field
* @property {boolean} translate If the field should be translated or not.
* @property {String} note The note for the translator.
*/
/**
* @typedef {Object} field
* @memberof SampleJobManager.contentType
* @property {string} name The name of the field.
* @property {SampleJobManager.contentType.field.translation} The translation object of the field.
*/
/**
* @typedef {Object} contentType
* @memberof SampleJobManager
* @property {string} contentType.name The name of the content type.
* @property {SampleJobManager.contentType.field[]} contentType.fields The fields of the content type.
*/
/**
* @typedef {Object} JobDetails
* @memberof SampleJobManager
* @property {object} assets Assets resources to translate
* @property {object} assets.jobJSON The job.json object for assets.
* @property {SampleJobManager.file[]} assets.files The array of files that need to be translated
* @property {SampleJobManager.contentType[]} assets.contentTypes The array of content types.
* @property {object} site Site resources to translate
* @property {object} site.jobJSON The job.json object for site resources.
* @property {SampleJobManager.file[]} site.files The array of files that need to be translated
* @property {string} sourceLanguage The source language for the job.
* @property {String[]} targetLanguages Array of target languages the resources will be translated into.
*/
/**
* Translate a job whose zip file has been imported into the persistence store. <br/>
* This first checks the current state of the job and executes any further steps required.
* <ul>
* <li>Import all files into the LSP: status updated to "TRANSLATING" (% complete set at 0%)</li>
* <li>Request files are translated into the target languages</li>
* <li>Poll until complete: status updated to "DOWNLOADING" (% complete updated also added during polling - stops at 99%)</li>
* <li>Download files: status updated to "TRANSLATED" (% complete updated to 100%)</li>
* <li>If any steps error: status is updated to "FAILED"</li>
* </ul>
* @param {SampleJobManager.JobConfig} jobConfig - The configuration of the connector job to run. This information is held as metadata in the connector for the job.
*/
SampleJobManager.prototype.translateJob = function (jobConfig) {
var self = this;
// get the job details
self.getJobDetails(jobConfig).then(function (jobDetails) {
var jobSteps = [];
// get all the supported target languages
var targetLanguages = jobDetails.targetLanguages.filter(function (language) {
return language !== null && language !== jobDetails.sourceLanguage;
});
// if job is not translated or failed
if (['FAILED', 'TRANSLATED'].indexOf(jobConfig.status) === -1) {
if (jobDetails.sourceLanguage === null) {
// source language is not supported by the connector
console.log('sampleJobManager.translateJob(): source locale: "' + jobDetails.origSourceLanguage + '" is not supported by the connector.');
// update status to failed
jobConfig.status = 'FAILED';
jobSteps.push(function () {
return self.updateStatus(jobConfig, 'FAILED');
});
} else if (targetLanguages.length === 0) {
// nothing to do, either no supported locales or only equivalent target locales ('es' ==> 'es-ES')
// update status to DOWNLOADING to skip IMPORTING/TRANSLATING steps below
jobConfig.status = 'DOWNLOADING';
}
}
// allow re-starting of a job.
// Only execute steps not already executed based on the job status
switch (['CREATED', 'IMPORTING', 'TRANSLATING', 'DOWNLOADING', 'TRANSLATED', 'FAILED'].indexOf(jobConfig.status)) {
case 0: // CREATED
console.log('sampleJobManager.translateJob(): unable to start job - ' + jobConfig.name + ' zip file not available');
return;
case 1: // IMPORTING
// import all the files into LSP server
jobSteps.push(function () {
return fileImporter.importFiles(jobConfig, jobDetails);
});
// update status to TRANSLATING
jobSteps.push(function () {
return self.updateStatus(jobConfig, "TRANSLATING");
});
case 2: // TRANSLATING
// tell the LSP provider to translate the files into the requested locales
jobSteps.push(function () {
return fileTranslator.translateFiles(jobConfig, jobDetails);
});
// wait until the LSP provider project is 100% complete
jobSteps.push(function () {
return self.pollStatus(jobConfig, jobDetails);
});
// update status to DOWNLOADING
jobSteps.push(function () {
return self.updateStatus(jobConfig, "DOWNLOADING");
});
case 3: // DOWNLOADING
// download the translated files
jobSteps.push(function () {
return fileDownloader.downloadFiles(jobConfig, jobDetails);
});
// now create the zip for the translated files
jobSteps.push(function () {
return self.createZip(jobConfig, jobDetails);
});
// update progress to 100%
jobSteps.push(function () {
return self.updateProgress(jobConfig, "100");
});
// update status to TRANSLATED
jobSteps.push(function () {
return self.updateStatus(jobConfig, "TRANSLATED");
});
case 4: // TRANSLATED
// already translated, we're done
break;
case 5: // FAILED
// job failed, nothing further we can do
break;
default:
// unable to restart this job
console.log('sampleJobManager.translateJob(): unable to restart job - ' + jobConfig.name + ' from status - ' + jobConfig.status);
return;
}
// now run through all the steps identified
// chain the promises in the array so that they execute as: return p1.then(return p2.then(return p3.then(...)));
var doTranslateJob = jobSteps.reduce(function (previousPromise, nextPromise) {
return previousPromise.then(function () {
// wait for the previous promise to complete and then return a new promise for the next
return nextPromise();
});
},
// Start with a previousPromise value that is a resolved promise
Promise.resolve());
// once all steps have completed, job is translated
doTranslateJob.then(function () {
console.log('sampleJobManager.translateJob(): Completed translation of job - ' + jobConfig.name);
}).catch(function (e) {
console.log('SampleJobManager.translateJob(): failed to translate job - ' + jobConfig.name);
var cyclicEntry = [];
console.log(JSON.stringify(e, function (key, val) {
if (val !== null && typeof val === "object") {
if (cyclicEntry.indexOf(val) >= 0) {
return;
}
cyclicEntry.push(val);
}
return val;
}));
// update the job status to 'FAILED' so we don't try and re-do this job, they need to re-submit it
self.updateStatus(jobConfig, "FAILED", e);
});
}).catch(function (error) {
console.log('sampleJobManager.translateJob(): failed to get job details for - ' + jobConfig.name);
return;
});
};
/**
* Get the job details from the job.json file included in the source zip file<br/>
* <br/>
* Within the job.json file, contentTypes node provides metadata of each content type and
* contentItems node lists the assets.<br/>
* <pre>
* "contentTypes": [
* {
* "name": {content type name}
* "fields": [
* "{field name}": {
* "caas-translation": {
* "translate": true | false // whether the field should be translated or not
* }
* },...
* ],
* "systemFields": [
* "file": {
* "caas-translation": {
* "translate": true | false // whether the native file of the asset should be translated or not
* }
* },...
* ]
* },...
* ],
* "contentItems": [
* {
* "name": {asset name},
* "id": {GUID},
* "type": {content type name}
* },...
* ]
* </pre>
* The folder structure differs between job types - the job types are: <br/>
* An "assets" job type:
* <ul>
* <li>job.json</li>
* <li>root</li>
* <ul>
* <li>[GUID-{name}.json,...]</li>
* <li>[files]
* <ul>
* <li>[GUID,...]
* <ul>
* <li>[{binary-file}]</li>
* </ul>
* </li>
* </ul>
* </li>
* </ul>
* </ul>
* Or a "site" job type:
* <ul>
* <li> assets
* <ul>
* <li>job.json</li>
* <li>root</li>
* <ul>
* <li>[GUID-{name}.json,...]</li>
* <li>[files]
* <ul>
* <li>[GUID,...]
* <ul>
* <li>[{binary-file}]</li>
* </ul>
* </li>
* </ul>
* </li>
* </ul>
* </ul>
* </li>
* <li> site
* <ul>
* <li>job.json</li>
* <li>root</li>
* </ul>
* </li>
* </ul>
* @param {SampleJobManager.JobConfig} jobConfig - The configuration of the connector job to run. This information is held as metadata in the connector for the job.
* @returns {Promise.<SampleJobManager.JobDetails>} A promise that contains the details of the combined job.json files
*/
SampleJobManager.prototype.getJobDetails = function (jobConfig) {
return new Promise(function (resolve, reject) {
var jobId = jobConfig.properties.id;
persistenceStore.getSourceJobJSON({
jobId: jobId
}).then(function (jobFiles) {
var jobDetails = {
type: jobFiles.type
},
folders = [],
assetJobJSON,
siteJobJSON;
// include the asset files
if (jobFiles.assetsJSON) {
try {
assetJobJSON = JSON.parse(jobFiles.assetsJSON);
jobDetails.assets = {
jobJSON: assetJobJSON,
files: [],
binaryFiles: [],
contentTypes: []
};
// Content type metadata
for (const jobContentType of (assetJobJSON.contentTypes || [])) {
var contentType = {
name: jobContentType.name,
fields: [],
systemFields: []
};
// fields property is introduced in 20.1.3.
if (jobContentType.hasOwnProperty('fields')) {
Object.keys(jobContentType.fields).forEach(function(key) {
var jobField = jobContentType.fields[key];
// If 'caas-translation' property exists, then create field object.
if (jobField.hasOwnProperty('caas-translation')) {
var field = {
name: jobField.name,
translation: {
translate: jobField['caas-translation'].translate,
note: jobField['caas-translation'].note
}
};
contentType.fields.push(field);
}
});
if (contentType.fields.length > 0) {
jobDetails.assets.contentTypes.push(contentType);
}
}
// systemFields property is introduced in 22.1.1.
if (jobContentType.hasOwnProperty('systemFields')) {
// Look for file. Other system fields will be consumed in the future.
if (jobContentType.systemFields.file) {
var systemField = {
name: jobContentType.systemFields.file.name,
translation: {
translate: jobContentType.systemFields.file['caas-translation'].translate,
note: jobContentType.systemFields.file['caas-translation'].note
}
};
contentType.systemFields.push(systemField);
}
}
}
// list all the asset files (including the file name)
for (const contentItem of (assetJobJSON.contentItems || [])) {
jobDetails.assets.files.push({
name: contentItem.name,
id: contentItem.id,
path: contentItem.id + '-' + contentItem.name + '.json'
});
// Check if the content item is of a type with translatable binary file.
// To be safe, check that the type field, which is new, exists first.
if (contentItem.type) {
// Look up the type object
var contentItemType = jobDetails.assets.contentTypes.find(function(contentType) {
return contentType.name === contentItem.type;
});
if (contentItemType) {
// Look up the file object in system field
var contentItemTypeFile = contentItemType.systemFields.find(function(field) {
return field.name === 'file';
});
if (contentItemTypeFile && contentItemTypeFile.translation.translate) {
// Each item represents a binary files folder to be discovered.
folders.push({
id: contentItem.id,
folderPath: persistenceStore.getBinaryFilePath({ contentItemId: contentItem.id})
});
}
}
}
}
// Discovery of binary files of the digital asset in the list of folders.
// There should be one binary file per folder, but pave the way for having multiple files now.
// List all binary files in the files/<content-id> folders.
var allBinaryFiles = [];
for (const folder of (folders || [])) {
var files = persistenceStore.getFolderFiles({
jobId: jobConfig.properties.id,
jobType: jobConfig.jobType,
folderPath: folder.folderPath,
fileType: 'assets',
id: folder.id
});
allBinaryFiles = allBinaryFiles.concat(files);
}
jobDetails.assets.binaryFiles = jobDetails.assets.binaryFiles.concat(allBinaryFiles);
} catch (e) {
console.log('sampleJobManager.getJobDetails(): failed to parse asset job.json file for - ' + jobId);
console.log(e);
return reject(e);
}
}
// include the site files
if (jobFiles.siteJSON) {
persistenceStore.getSourceFile({
jobId: jobId,
fileType: 'site',
filePath: 'structure.json'
}).then(function (siteStructureFile) {
try {
var siteStructure = JSON.parse(siteStructureFile);
siteJobJSON = JSON.parse(jobFiles.siteJSON);
jobDetails.site = {
jobJSON: siteJobJSON,
files: []
};
// add in the pages
for (const pageItem of (siteJobJSON.pages || [])) {
var pageId = pageItem.name.replace('.json', ''),
pageEntry;
// find the entry for this page so we use the menu name rather than id for display
pageEntry = (siteStructure.pages || []).find(function (page) {
return page.id.toString() === pageId;
});
if (!pageEntry) {
pageEntry = {
"id": pageId,
"name": pageItem.name
};
}
jobDetails.site.files.push({
name: 'Page:' + pageEntry.name,
id: pageItem.id,
path: pageItem.name
});
}
// add in the structure/siteinfo
jobDetails.site.files.push({
name: 'Site Menu',
id: 'structure',
path: 'structure.json'
});
jobDetails.site.files.push({
name: 'Site Information',
id: 'siteinfo',
path: 'siteinfo.json'
});
} catch (e) {
console.log('sampleJobManager.getJobDetails(): failed to parse asset job.json file for - ' + jobId);
return reject(e);
}
// get the source & target languages for site and assets (will be the same)
jobDetails.origSourceLanguage = siteJobJSON.sourceLanguage;
jobDetails.origTargetLanguages = siteJobJSON.targetLanguages;
// update locales to those supported by the LSP
jobDetails.sourceLanguage = localeMap.getLSPLocale(siteJobJSON.sourceLanguage);
jobDetails.targetLanguages = siteJobJSON.targetLanguages.map(function (targetLanguage) {
return localeMap.getLSPLocale(targetLanguage);
});
// remove any duplicates from the target languages array
// e.g.:
// ['de-DE', 'de-DE-x-bayern'] would map to: ['de-DE', 'de-DE']
// ['en', 'en-US'] would map to: ['en-US', 'en-US']
jobDetails.targetLanguages = [...new Set(jobDetails.targetLanguages)];
return resolve(jobDetails);
});
} else {
// get the source & target languages for assets only
jobDetails.origSourceLanguage = assetJobJSON.sourceLanguage;
jobDetails.origTargetLanguages = assetJobJSON.targetLanguages;
// update locales to those supported by the LSP
jobDetails.sourceLanguage = localeMap.getLSPLocale(assetJobJSON.sourceLanguage);
jobDetails.targetLanguages = assetJobJSON.targetLanguages.map(function (targetLanguage) {
return localeMap.getLSPLocale(targetLanguage);
});
// remove any duplicates from the target languages array
// e.g.:
// ['de-DE', 'de-DE-x-bayern'] would map to: ['de-DE', 'de-DE']
// ['en', 'en-US'] would map to: ['en-US', 'en-US']
jobDetails.targetLanguages = [...new Set(jobDetails.targetLanguages)];
return resolve(jobDetails);
}
}).catch(function (error) {
console.log('sampleJobManager.getJobDetails(): unable to get job.json files for job - ' + jobId);
return reject(error);
});
});
};
/**
* Update the job status in the job metadata.
* @param {SampleJobManager.JobConfig} jobConfig - The configuration of the connector job to run. This information is held as metadata in the connector for the job.
* @property {('CREATED'|'IMPORTING'|'TRANSLATING'|'DOWNLOADING'|'TRANSLATED')} status The new status of the job.
* @property {string} statusMessage Any message to include with the status update.
* @returns {Promise.<SampleJobManager.JobConfig>} A promise that contains the details of the updated job metadata
*/
SampleJobManager.prototype.updateStatus = function (jobConfig, status, statusMessage) {
// get the current job metadata
return persistenceStore.getJob({
jobId: jobConfig.properties.id
}).then(function (jobMetadata) {
// update the status value in the job metadata
jobMetadata.status = status;
jobMetadata.statusMessage = statusMessage;
return persistenceStore.updateJob(jobMetadata);
});
};
/**
* Update the job progress in the job metadata.
* @param {SampleJobManager.JobConfig} jobConfig - The configuration of the connector job to run. This information is held as metadata in the connector for the job.
* @property {number} progress New percentage progress value.
* @returns {Promise.<SampleJobManager.JobConfig>} A promise that contains the details of the updated job metadata
*/
SampleJobManager.prototype.updateProgress = function (jobConfig, progress) {
return persistenceStore.getJob({
jobId: jobConfig.properties.id
}).then(function (jobMetadata) {
// update the progress
jobMetadata.progress = jobConfig.progress = progress;
return persistenceStore.updateJob(jobMetadata);
});
};
/**
* Background task to poll the Language Service Provider and wait for the job to be 100% complete.
* @param {SampleJobManager.JobConfig} jobConfig - The configuration of the connector job to run. This information is held as metadata in the connector for the job.
* @param {SampleJobManager.JobDetails} jobDetails - The details of the combined job.json files.
* @returns {Promise} A promise that resolves when polling gets to 100%.
*/
SampleJobManager.prototype.pollStatus = function (jobConfig, jobDetails) {
var self = this;
return new Promise(function (resolve, reject) {
pollServer = function () {
// get the current status for the job
translationProvider.getProjectStatus(jobConfig.authToken, jobConfig.properties.projectId).then(function (projectStatus) {
// if projectStatus complete...
if (projectStatus.properties.progress.toString() === "100") {
// update status to "DOWNLOADING"
self.updateStatus(jobConfig, "DOWNLOADING", "Downloading translated files").then(function () {
self.updateProgress(jobConfig, "95").then(function (jobMetadata) {
// resolve
resolve();
}).catch(function (error) {
console.log('sampleJobManager.pollStatus(): failed to update progress for job - ' + jobConfig.properties.id);
reject(error);
});
}).catch(function (error) {
console.log('sampleJobManager.pollStatus(): failed to update status for job - ' + jobConfig.properties.id);
reject(error);
});
} else {
// Show the actual progress returned by the LSP.
progress = projectStatus.properties.progress;
self.updateProgress(jobConfig, progress).then(function (jobMetadata) {
// wait and poll again
setTimeout(function () {
pollServer();
}, 5000);
});
}
}).catch(function (e) {
// failed to get project status - this may be fatal, would need to confirm project exists
// update status to "FAILED"
self.updateStatus(jobConfig, "FAILED", "Failed to get status for project - " + jobConfig.properties.projectId);
reject();
});
};
// kick off the polling for this job
pollServer();
});
};
/**
* Create a zip file of all the downloaded translations.
* @param {SampleJobManager.JobConfig} jobConfig - The configuration of the connector job to run. This information is held as metadata in the connector for the job.
* @param {SampleJobManager.JobDetails} jobDetails - The details of the combined job.json files.
* @returns {Promise.<SampleJobManager.JobConfig>} A promise that contains the job metadata updated with the location of the translation zip file.
*/
SampleJobManager.prototype.createZip = function (jobConfig, jobDetails) {
return persistenceStore.createTranslationZip({
jobId: jobConfig.properties.id
});
};
// Export the mock translator
module.exports = new SampleJobManager();
