Outputting to the frontend

In sift.json, add a new entry to the Sift outputs section like this:

{
  // ...
    "outputs": {
      "exports": {
        "count": {
          "key$schema":"string"
        },
        "emails": {
          "key$schema":"string"
        }
      }
    }
  // ...
}

An export is data that we want to leave the confines of the server side code. Anything sent here will be available for API query and synchronized to the frontend in a key / value pair form. We are calling our exports count and emails, and we defined a key schema that is a single string. All this means that the system will expect a flat key for the data in this bucket.

Let’s go back to the "Parse Emails Node" node definition and make it reference count and emails. Create a new node for aggregating the total counts, server/count.js .

{
  // ...
  "dag": {
  // ...
    "nodes": [{
      "#": "Parse Emails Node",
      "input": {
        "bucket": "gmailEmails"
      },
      "implementation": {
        "javascript": "server/parser.js"
      },
      "outputs": {
        "emails-st": {},
        "emails": {}
      }
    }, {
      "#": "Counts Total Node",
      "input": {
        "bucket": "emails-st"
      },
      "implementation": {
        "javascript": "server/count.js"
      },
      "outputs": {
        "count": {}
      }
    }],
    "stores": {
      "emails-st": {}
    },
    "outputs": {
      "exports": {
        "count": {
          "key$schema": "string"
        },
        "emails": {
          "key$schema": "string"
        }
      }
    }
  }
}

Now email data is set up to be processed through server/parser.js and exported out via emails and to the emails-st store for further processing in the server/count.js node.

Let’s modify our code in parser.js to output redacted email data for the frontend list. We'll create a count.js node for aggregating the counts and output them.

// ...
module.exports = function (got) {
  // ...
  
  // replace: return emails.map(...);

  const response = emails.map((email) => {
    // ..
    const { id } = email;
    const key = id;
    const value = { text, count, subject, id };
    
    return [{
        name: 'emails-st', // back-end store for further processing in `count.js`
      key,
      value
    }, {
      name: 'emails', // front-end output
      key,
      value: { count, subject, id } // redact text from client-side
    }];
  };
  
  // flatten the node output into a flat array ([[0, 1], [2, 3]] -> [0, 1, 2, 3]
  return [].concat(...response);

// ...
module.exports = function (got) {
  const inData = got.in;
  const emails = inData.data.map(({ key, value }) => {
    try {
      return { key, value: JSON.parse(value) };
    }
    catch (err) {
      console.error('count.js: something went wrong with input:', err);
      return null;
    }
  }).filter(i => i);

  const totalWordCount = messages
    .map(({ value: { count } }) => count)
    .reduce((p, c) => p + c, 0);

  // where `count` is a front-end output
  return [
    { name: 'count', key: 'TOTAL_EMAILS', value: emails.length },
    { name: 'count', key: 'TOTAL_WORDS', value: totalWordCount },
  ];
};