Merge pull request #753 from topcoder-platform/feature/plat-3292

use the new file scanner service to av scan attachments

Author: ThomasKranitsas

config/default.json

@@ -14,6 +14,8 @@
   "directProjectServiceEndpoint": "",
   "directProjectServiceTimeout": 5000,
   "attachmentsS3Bucket": "topcoder-prod-media",
+  "attachmentsDMZS3Bucket": "topcoder-prod-media-dmz",
+  "attachmentsQuarantineS3Bucket": "topcoder-prod-media-quarantine",
   "projectAttachmentPathPrefix": "projects",
   "projectAttachmentPathSuffix": "attachments",
   "elasticsearchConfig": {
@@ -87,4 +89,4 @@
   },
   "STRIPE_SECRET_KEY": "",
   "sfdcBillingAccountNameField": "Billing_Account_Name__c"
-}
+}

config/development.json

@@ -2,6 +2,8 @@
   "pubsubQueueName": "dev.project.service",
   "pubsubExchangeName": "dev.projects",
   "attachmentsS3Bucket": "topcoder-dev-media",
+  "attachmentsDMZS3Bucket": "topcoder-dev-media-dmz",
+  "attachmentsQuarantineS3Bucket": "topcoder-dev-media-quarantine",
   "connectProjectsUrl": "https://connect.topcoder-dev.com/projects/",
   "fileServiceEndpoint": "https://api.topcoder-dev.com/v3/files/",
   "connectProjectsUrl": "https://connect.topcoder-dev.com/projects/",

src/constants.js

@@ -139,6 +139,8 @@ export const EVENT = {
 };
 
 export const BUS_API_EVENT = {
+  AV_SCAN_REQUEST: 'avscan.action.scan',
+
   PROJECT_CREATED: 'project.action.create',
   PROJECT_UPDATED: 'project.action.update',
   PROJECT_DELETED: 'project.action.delete',
@@ -151,6 +153,7 @@ export const BUS_API_EVENT = {
   PROJECT_ATTACHMENT_ADDED: 'project.action.create',
   PROJECT_ATTACHMENT_REMOVED: 'project.action.delete',
   PROJECT_ATTACHMENT_UPDATED: 'project.action.update',
+  PROJECT_ATTACHMENT_SCAN_RESULT: 'avscan.projects.assets.result',
 
   // When phase is added/updated/deleted from the project,
   // When product is added/deleted from a phase

src/events/attachments/index.js

@@ -0,0 +1,88 @@
+/**
+ * Event handlers for attachment create, and av scan result
+ */
+import _ from 'lodash';
+import Joi from 'joi';
+import Promise from 'bluebird';
+import config from 'config';
+import util from '../../util';
+import { BUS_API_EVENT } from '../../constants';
+import { createEvent } from '../../services/busApi';
+
+/**
+ * Payload for new unified BUS events like `avscan.projects.assets.result` with `resource=attachment`
+ */
+const attachmentScanResultPayloadSchema = Joi.object().keys({
+  url: Joi.string().required(),
+  fileName: Joi.string().required(),
+  isInfected: Joi.boolean().required(),
+}).unknown(true).required();
+
+/**
+ * Updates project activity fields. throws exceptions in case of error
+ * @param   {Object}  app       Application object used to interact with RMQ service
+ * @param   {String}  topic     Kafka topic
+ * @param   {Object}  payload   Message payload
+ * @return  {Promise} Promise
+ */
+async function attachmentScanResultKafkaHandler(app, topic, payload) {
+  // Validate payload
+  const result = Joi.validate(payload, attachmentScanResultPayloadSchema);
+  if (result.error) {
+    throw new Error(result.error);
+  }
+  const sourceBucket = config.get('attachmentsDMZS3Bucket');
+  // if the attachment is infected, move it to the quarantine s3 bucket, if not move it to the clean s3 bucket
+  if (payload.isInfected) {
+    // move to quarantine
+    const destBucket = config.get('attachmentsQuarantineS3Bucket');
+    util.s3FileTransfer({ log: app.logger }, sourceBucket, payload.path, destBucket, payload.path)
+    app.logger.debug(`Attachment ${payload.fileName} is infected, moving to quarantine`);
+  } else {
+    // move to clean
+    const destBucket = config.get('attachmentsS3Bucket');
+    util.s3FileTransfer({ log: app.logger }, sourceBucket, payload.path, destBucket, payload.path)
+  }
+}
+
+/**
+ * Payload for new unified BUS events like `avscan.action.scan` with `resource=attachment`
+ */
+const attachmentPayloadSchema = Joi.object().keys({
+  path: Joi.string().required(),
+}).unknown(true).required();
+
+/**
+ * Attachment Created BUS API event handler.
+ * - requests av scan by posting a kafka message to `avscan.action.scan` topic
+ * - throws exceptions in case of error
+ *
+ * @param   {Object}  app       Application object
+ * @param   {String}  topic     Kafka topic
+ * @param   {Object}  payload   Message payload
+ * @return  {Promise} Promise
+ */
+async function attachmentCreatedKafkaHandler(app, topic, payload) {
+  // Validate payload
+  const result = Joi.validate(payload, attachmentPayloadSchema);
+  if (result.error) {
+    throw new Error(result.error);
+  }
+  const avScanPayload = {
+    url: payload.path,
+    fileName: payload.path.split('/').pop(),
+    moveFile: false,
+    callbackOption: 'kafka',
+    callbackKafkaTopic: BUS_API_EVENT.PROJECT_ATTACHMENT_SCAN_RESULT,
+  };
+  await createEvent(
+    BUS_API_EVENT.AV_SCAN_REQUEST,
+    avScanPayload,
+    app.logger,
+  )
+}
+
+module.exports = {
+  attachmentScanResultKafkaHandler,
+  attachmentCreatedKafkaHandler,
+};

src/events/kafkaHandlers.js

@@ -17,6 +17,10 @@ import {
 } from './projectPhases';
 import { timelineAdjustedKafkaHandler } from './timelines';
 import { milestoneUpdatedKafkaHandler } from './milestones';
+import {
+  attachmentScanResultKafkaHandler,
+  attachmentCreatedKafkaHandler,
+} from './attachments'
 
 const kafkaHandlers = {
   /**
@@ -37,6 +41,9 @@ const kafkaHandlers = {
   // Events coming from timeline/milestones (considering it as a separate module/service in future)
   [CONNECT_NOTIFICATION_EVENT.MILESTONE_TRANSITION_COMPLETED]: milestoneUpdatedKafkaHandler,
   [CONNECT_NOTIFICATION_EVENT.TIMELINE_ADJUSTED]: timelineAdjustedKafkaHandler,
+
+  // Events coming from attachments
+  [BUS_API_EVENT.PROJECT_ATTACHMENT_SCAN_RESULT]: attachmentScanResultKafkaHandler,
 };
 
 /**
@@ -95,6 +102,10 @@ registerKafkaHandler(
   RESOURCES.PHASE,
   projectPhaseRemovedKafkaHandler,
 );
-
+registerKafkaHandler(
+  BUS_API_EVENT.PROJECT_ATTACHMENT_ADDED,
+  RESOURCES.ATTACHMENT,
+  attachmentCreatedKafkaHandler,
+);
 
 export default kafkaHandlers;