implemented amplitude metrics (#1141)

server/amplitude.js

@@ -0,0 +1,161 @@
+const crypto = require('crypto');
+const geoip = require('fxa-geodb')();
+const fetch = require('node-fetch');
+const config = require('./config');
+const pkg = require('../package.json');
+
+const HOUR = 1000 * 60 * 60;
+
+function truncateToHour(timestamp) {
+  return Math.floor(timestamp / HOUR) * HOUR;
+}
+
+function orderOfMagnitude(n) {
+  return Math.floor(Math.log10(n));
+}
+
+function userId(fileId, ownerId) {
+  const hash = crypto.createHash('sha256');
+  hash.update(fileId);
+  hash.update(ownerId);
+  return hash.digest('hex').substring(32);
+}
+
+function location(ip) {
+  try {
+    return geoip(ip);
+  } catch (e) {
+    return {};
+  }
+}
+
+function statUploadEvent(data) {
+  const loc = location(data.ip);
+  const event = {
+    session_id: -1,
+    country: loc.country,
+    region: loc.state,
+    user_id: userId(data.id, data.owner),
+    app_version: pkg.version,
+    time: truncateToHour(Date.now()),
+    event_type: 'server_upload',
+    user_properties: {
+      download_limit: data.dlimit,
+      time_limit: data.timeLimit,
+      size: orderOfMagnitude(data.size),
+      anonymous: data.anonymous
+    },
+    event_id: 0
+  };
+  return sendBatch([event]);
+}
+
+function statDownloadEvent(data) {
+  const loc = location(data.ip);
+  const event = {
+    session_id: -1,
+    country: loc.country,
+    region: loc.state,
+    user_id: userId(data.id, data.owner),
+    app_version: pkg.version,
+    time: truncateToHour(Date.now()),
+    event_type: 'server_download',
+    event_properties: {
+      download_count: data.download_count,
+      ttl: data.ttl
+    },
+    event_id: data.download_count
+  };
+  return sendBatch([event]);
+}
+
+function statDeleteEvent(data) {
+  const loc = location(data.ip);
+  const event = {
+    session_id: -1,
+    country: loc.country,
+    region: loc.state,
+    user_id: userId(data.id, data.owner),
+    app_version: pkg.version,
+    time: truncateToHour(Date.now()),
+    event_type: 'server_delete',
+    event_properties: {
+      download_count: data.download_count,
+      ttl: data.ttl
+    },
+    event_id: data.download_count + 1
+  };
+  return sendBatch([event]);
+}
+
+function clientEvent(event, ua, language, session_id, deltaT, platform, ip) {
+  const loc = location(ip);
+  const ep = event.event_properties || {};
+  const up = event.user_properties || {};
+  const event_properties = {
+    browser: ua.browser.name,
+    browser_version: ua.browser.version,
+    status: ep.status,
+
+    age: ep.age,
+    downloaded: ep.downloaded,
+    download_limit: ep.download_limit,
+    duration: ep.duration,
+    file_count: ep.file_count,
+    password_protected: ep.password_protected,
+    size: ep.size,
+    time_limit: ep.time_limit,
+    trigger: ep.trigger,
+    ttl: ep.ttl
+  };
+  const user_properties = {
+    active_count: up.active_count,
+    anonymous: up.anonymous,
+    experiments: up.experiments,
+    first_action: up.first_action
+  };
+  return {
+    app_version: pkg.version,
+    country: loc.country,
+    device_id: event.device_id,
+    event_properties,
+    event_type: event.event_type,
+    language,
+    os_name: ua.os.name,
+    os_version: ua.os.version,
+    platform,
+    region: loc.state,
+    session_id,
+    time: event.time + deltaT,
+    user_id: event.user_id,
+    user_properties
+  };
+}
+
+async function sendBatch(events, timeout = 1000) {
+  if (!config.amplitude_id) {
+    return 200;
+  }
+  try {
+    const result = await fetch('https://api.amplitude.com/batch', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        api_key: config.amplitude_id,
+        events
+      }),
+      timeout
+    });
+    return result.status;
+  } catch (e) {
+    return 500;
+  }
+}
+
+module.exports = {
+  statUploadEvent,
+  statDownloadEvent,
+  statDeleteEvent,
+  clientEvent,
+  sendBatch
+};

server/config.js

@@ -80,6 +80,11 @@ const conf = convict({
     arg: 'port',
     env: 'PORT'
   },
+  amplitude_id: {
+    format: String,
+    default: '',
+    env: 'AMPLITUDE_ID'
+  },
   analytics_id: {
     format: String,
     default: '',

server/initScript.js

@@ -24,11 +24,6 @@ var SENTRY_ID = '${config.sentry_id}';
 `;
 }
 
-let ga = '';
-if (config.analytics_id) {
-  ga = `var GOOGLE_ANALYTICS_ID = '${config.analytics_id}';`;
-}
-
 module.exports = function(state) {
   const authConfig = state.authConfig
     ? `var AUTH_CONFIG = ${JSON.stringify(state.authConfig)};`
@@ -71,7 +66,6 @@ module.exports = function(state) {
     state.downloadMetadata ? raw(JSON.stringify(state.downloadMetadata)) : '{}'
   };
   ${authConfig};
-  ${ga}
   ${sentry}
   `;
   return state.cspNonce

server/routes/delete.js

@@ -1,9 +1,20 @@
 const storage = require('../storage');
+const { statDeleteEvent } = require('../amplitude');
 
 module.exports = async function(req, res) {
   try {
-    await storage.del(req.params.id);
+    const id = req.params.id;
+    const meta = req.meta;
+    const ttl = await storage.ttl(id);
+    await storage.del(id);
     res.sendStatus(200);
+    statDeleteEvent({
+      id,
+      ip: req.ip,
+      owner: meta.owner,
+      download_count: meta.dl,
+      ttl
+    });
   } catch (e) {
     res.sendStatus(404);
   }

server/routes/download.js

@@ -1,6 +1,7 @@
 const storage = require('../storage');
 const mozlog = require('../log');
 const log = mozlog('send.download');
+const { statDownloadEvent } = require('../amplitude');
 
 module.exports = async function(req, res) {
   const id = req.params.id;
@@ -21,6 +22,14 @@ module.exports = async function(req, res) {
 
       const dl = meta.dl + 1;
       const dlimit = meta.dlimit;
+      const ttl = await storage.ttl(id);
+      statDownloadEvent({
+        id,
+        ip: req.ip,
+        owner: meta.owner,
+        download_count: dl,
+        ttl
+      });
       try {
         if (dl >= dlimit) {
           await storage.del(id);

server/routes/index.js

@@ -1,6 +1,7 @@
 const crypto = require('crypto');
 const express = require('express');
 const helmet = require('helmet');
+const uaparser = require('ua-parser-js');
 const storage = require('../storage');
 const config = require('../config');
 const auth = require('../middleware/auth');
@@ -12,6 +13,7 @@ const IS_DEV = config.env === 'development';
 const ID_REGEX = '([0-9a-fA-F]{10})';
 
 module.exports = function(app) {
+  app.set('trust proxy', true);
   app.use(helmet());
   app.use(
     helmet.hsts({
@@ -19,6 +21,10 @@ module.exports = function(app) {
       force: !IS_DEV
     })
   );
+  app.use(function(req, res, next) {
+    req.ua = uaparser(req.header('user-agent'));
+    next();
+  });
   app.use(function(req, res, next) {
     req.cspNonce = crypto.randomBytes(16).toString('hex');
     next();
@@ -35,12 +41,10 @@ module.exports = function(app) {
             'wss://send.firefox.com',
             'https://*.dev.lcip.org',
             'https://*.accounts.firefox.com',
-            'https://sentry.prod.mozaws.net',
-            'https://www.google-analytics.com'
+            'https://sentry.prod.mozaws.net'
           ],
           imgSrc: [
             "'self'",
-            'https://www.google-analytics.com',
             'https://*.dev.lcip.org',
             'https://firefoxusercontent.com'
           ],
@@ -92,7 +96,7 @@ module.exports = function(app) {
     require('./params')
   );
   app.post(`/api/info/:id${ID_REGEX}`, auth.owner, require('./info'));
-
+  app.post('/api/metrics', require('./metrics'));
   app.get('/__version__', function(req, res) {
     res.sendFile(require.resolve('../../dist/version.json'));
   });

server/routes/metrics.js

@@ -0,0 +1,23 @@
+const { sendBatch, clientEvent } = require('../amplitude');
+
+module.exports = async function(req, res) {
+  try {
+    const data = req.body;
+    const deltaT = Date.now() - data.now;
+    const events = data.events.map(e =>
+      clientEvent(
+        e,
+        req.ua,
+        data.lang,
+        data.session_id + deltaT,
+        deltaT,
+        data.platform,
+        req.ip
+      )
+    );
+    const status = await sendBatch(events);
+    res.sendStatus(status);
+  } catch (e) {
+    res.sendStatus(500);
+  }
+};

server/routes/ws.js

@@ -5,6 +5,7 @@ const mozlog = require('../log');
 const Limiter = require('../limiter');
 const wsStream = require('websocket-stream/stream');
 const fxa = require('../fxa');
+const { statUploadEvent } = require('../amplitude');
 
 const { Duplex } = require('stream');
 
@@ -105,6 +106,15 @@ module.exports = function(ws, req) {
         // in order to avoid having to check socket state and clean
         // up storage, possibly with an exception that we can catch.
         ws.send(JSON.stringify({ ok: true }));
+        statUploadEvent({
+          id: newId,
+          ip: req.ip,
+          owner,
+          dlimit,
+          timeLimit,
+          anonymous: !user,
+          size: limiter.length
+        });
       }
     } catch (e) {
       log.error('upload', e);