Configuration Files for Contentstack DataSync

The configuration file contains plain text parameters that define settings or preferences for Contentstack DataSync. This file will be referenced by the modules for performing the overall sync operations.

Config for Webhook Listener:

Configs for DataSync Manager:

Configs for DataSync Asset Store Filesystem

Configs for DataSync Content Store Filesystem

Configs for DataSync Content Store MongoDB

Below are the configuration files for different elements of Contentstack DataSync. You can make changes to these files to meet your requirement.

Webhook Listener

listener: {
    port: 5000,
    endpoint: '/notify',
}

DataSync Manager

/**
 * Sync manager configurations
 */
const config = {
  // asset store configuration, refer assetStore config for further details
  assetStore: {},
  // content store configuration, refer mongoStore OR fsStore config for further details
  contentStore: {
    unwanted: {
      asset: {
        _in_progress: true,
        _type: true,
        content_type_uid: true,
        data: true,
        publish_details: true,
      },
      contentType: {
        content_type_uid: true,
      },
      entry: {
        _content_type: true,
        _in_progress: true,
        _type: true,
        content_type_uid: true,
        data: true,
        publish_details: true,
      },
    },
  },
  // contentstack config, this is used mainly in making API calls
  contentstack: {
    apiKey: '***',
    deliveryToken: '***',
    sync_token: '***',
    branch: 'main',
    MAX_RETRY_LIMIT: 6,
    actions: {
      delete: 'delete',
      publish: 'publish',
      unpublish: 'unpublish',
    },
    apis: {
      content_types: '/v3/content_types/',
      sync: '/v3/stacks/sync',
    },
    host: 'cdn.contentstack.io',
    options: 'ig',
    port: 443,
    protocol: 'https:',
    query: {
      include_global_field_schema: true,
    },
    regexp: {
      asset_pattern: {
        options: 'ig',
        url: 'https://(assets|images).contentstack.io/v3/assets/(.*?)/(.*?)/(.*?)/(.*)',
      },
      rte_asset_pattern_1: {
        options: 'ig',
        url: 'https://(assets|images).contentstack.io/v3/assets/(.*?)/(.*?)/(.*?)/(.*?)(?=")',
      },
      rte_asset_pattern_2: {
        options: 'g',
        url: 'https://(assets|images).contentstack.io/v3/assets/(.*?)/(.*?)/(.*?)/(.*?)(.*)',
      },
    },
    verbs: {
      get: 'GET',
    },
  },
  // producer, ex: contentstack-webhook-listener config, check listener for further details
  listener: {},
  // locales, that you'd want to be skipped, ex: [ { code: 'en-us' } ]
  locales: [],
  // sync-manager specific configurations
  syncManager: {
    cooloff: 3000,
    inet: {
      dns: '8.8.8.8',
      host: 'contentstack.io',
      // DNS port
      port: 53,
      retries: 5,
      retryIncrement: 5 * 1000,
      retryTimeout: 90 * 1000,
      timeout: 30 * 1000,
      type: 'A',
    },
    limit: 100,
    markdown: {
      breaks: true,
      gfm: true,
      smartLists: true,
      tables: true,
      xhtml: false,
    },
    // max file sizes in bytes
    maxsize: 2097152,
    // waits for 10s if the app runs into errors or a kill signal is sent
    processTimeout: 10 * 1000,
    queue: {
      pause_threshold: 7000,
      resume_threshold: 4000,
    },
    saveFailedItems: true,
    saveFilteredItems: true,
  },
  plugins: [
     // Plugins starting with _cs_internal_ are app specific internal plugins
     // If you have custom plugins
     //  1. Ceate a 'plugins' folder in this app's directory
     //  2. Create a folder inside it
     //  3. Inside the folder, create a index.js file
     // Check example - plugins/dummy-plugin of this project
     {
       disabled: true,
       name: 'item-logger',
       options: {
       // Here, you can set your own custom logger
       log: console.log
       }
     },
     {
       disabled: false,
       // This plugin
       //  - Modifies reference field structure of Stack's v3.4.x and below to new reference field structure (Array of uids, transformed to Array of objects)
       //  - Updates all old reference fields and makes them compatible with multiple references
       name: '_cs_internal_transform_entries',
       // path: '',
       options: {
         // other overrides...
       },
     },
     {
       disabled: false,
       // This plugin
       //  - Detects asset/reference paths in a content type's schema
       //  - Updates all content types with asset/reference fields
      // Which can be later used with datasync-filesystem-sdk OR datasync-mongodb-sdk
       name: '_cs_internal_transform_schemas',
       options: {
         // Set false, if you do not wish to log asset references in the schema
         logAssetPaths: true,
         // Set false, if you do not wish to log entry references in the schema
         logReferencePaths: true,
       },
     },
     {
       disabled: false,
       // This plugin
       //  - Detects RTE/Markdown fields in a schema
       //  - Finds all the assets that's stored in Contentstack
       //  - Downloads all assets onto the asset store specified
      //  - Updates the entry's RTE/Markdown fields
       name: '_cs_internal_save_rte_markdown_assets',
       options: {
         // other overrides...
       },
     },
   ],
}

DataSync Content Store Filesystem

'contentStore': {
    baseDir: './_contents',
    locale: 'en-us',
    projections: {
      _content_type_uid: 0,
    },
}

DataSync Asset Store Filesystem

'assetStore': {
    baseDir: './_contents'
}
// baseDir : file location to store the data

DataSync Content Store MongoDB

/**
 * MongoDB content store specific configurations
 */
const mongoStore = {
  // Name of the MongoDB database to store in
  dbName: 'contentstack-db',
  // Collection name in which contents (assets, schema, and entries) will be stored
  collection: {
      asset: 'contents',
      entry: 'contents',
      schema: 'contents',
    },
  // Refer http://mongodb.github.io/node-mongodb-native/3.1/api/MongoClient.html for more configuration options
  options: {
    autoReconnect: true,
    connectTimeoutMS: 15000,
    keepAlive: true,
    noDelay: true,
    reconnectInterval: 1000,
    reconnectTries: 20,
    userNewUrlParser: true,
  },
  // Keys to be deleted, while data is being inserted
  unwantedKeys: {
    asset: {
      action: true,
      checkpoint: true,
      'data.created_by': true,
      event_at: true,
      type: true,
      'data.updated_by': true
    },
    contentType: {
      'data.created_by': true,
      'data.updated_by': true,
      'data.DEFAULT_ACL': true,
      'data.SYS_ACL': true,
      'data.abilities': true,
      'data.last_activity': true
    },
    entry: {
      action: true,
      checkpoint: true,
      'data.created_by': true,
      event_at: true,
      type: true,
      'data.updated_by': true
    },
  },
  // mongodb connection url
  url: 'mongodb://localhost:27017',
}

DataSync Filesystem SDK

contentStore: {
     // baseDir : file location of stored data
      baseDir: './_contents',
    }
},
// locale info, control the locales to be supported by the SDK
  // ex: [ { code: 'en-us', relative_url_prefix: '/' } ]
  locales: [],

DataSync MongoDB SDK

/**
 * Contentstack sync mongodb SDK configurations
 */
const mongoSDK = {
  // Name of the mongodb database to store in
  dbName: 'contentstack-db',
  // Collection name in which content (assets, schema, and entries) will be stored
  collection: {
      asset: 'contents',
      entry: 'contents',
      schema: 'contents',
    },
  // Indexes to be applied
  indexes: {
      _content_type_uid: 1,
      locale: 1,
      uid: 1,
      updated_at: -1,
    },
  // Refer http://mongodb.github.io/node-mongodb-native/3.1/api/MongoClient.html for more configuration options
  options: {
    autoReconnect: true,
    connectTimeoutMS: 15000,
    keepAlive: true,
    noDelay: true,
    reconnectInterval: 1000,
    reconnectTries: 20,
    userNewUrlParser: true,
  },
  // mongodb connection url
  uri: 'mongodb://localhost:27017',
  // no of items to be returned by default, when a query is fired
  // use .limit(200) to override this
  limit: 100,
  // no of items to be skipped by default, when a query is fired
  // use .skip(20) to override this
  skip: 0,
  // locale info, control the locales to be supported by the SDK
  locales: 'en-us',
  // keys to be ignored by default
  // use .only(['_id', 'updated_at']) to override these configurations
 projections: {
      _content_type_uid: 0,
      _id: 0,
    },
}