integrate with new ui

server/storage/fs.js

@@ -6,7 +6,7 @@ const mkdirp = require('mkdirp');
 const stat = promisify(fs.stat);
 
 class FSStorage {
-  constructor(config, log) {
+  constructor(config, index, log) {
     this.log = log;
     this.dir = config.file_dir;
     mkdirp.sync(this.dir);

server/storage/index.js

@@ -5,10 +5,16 @@ const createRedisClient = require('./redis');
 
 class DB {
   constructor(config) {
-    const Storage = config.s3_bucket ? require('./s3') : require('./fs');
+    const Storage =
+      config.s3_buckets.length > 0 ? require('./s3') : require('./fs');
     this.log = mozlog('send.storage');
-    this.expireSeconds = config.expire_seconds;
-    this.storage = new Storage(config, this.log);
+
+    this.storage = [];
+
+    for (let i = 0; i < config.num_of_buckets; i++) {
+      this.storage.push(new Storage(config, i, this.log));
+    }
+
     this.redis = createRedisClient(config);
     this.redis.on('error', err => {
       this.log.error('Redis:', err);
@@ -20,32 +26,51 @@ class DB {
     return Math.ceil(result) * 1000;
   }
 
-  length(id) {
-    return this.storage.length(id);
+  async getBucket(id) {
+    return this.redis.hgetAsync(id, 'bucket');
   }
 
-  get(id) {
-    return this.storage.getStream(id);
+  async length(id) {
+    const bucket = await this.redis.hgetAsync(id, 'bucket');
+    return this.storage[bucket].length(id);
   }
 
-  async set(id, file, meta) {
-    await this.storage.set(id, file);
+  async get(id) {
+    const bucket = await this.redis.hgetAsync(id, 'bucket');
+    return this.storage[bucket].getStream(id);
+  }
+
+  async set(id, file, meta, expireSeconds = config.default_expire_seconds) {
+    const bucketTimes = config.expire_times_seconds;
+    let bucket = 0;
+    while (bucket < config.num_of_buckets - 1) {
+      if (expireSeconds <= bucketTimes[bucket]) {
+        break;
+      }
+      bucket++;
+    }
+
+    await this.storage[bucket].set(id, file);
+    this.redis.hset(id, 'bucket', bucket);
     this.redis.hmset(id, meta);
-    this.redis.expire(id, this.expireSeconds);
+    this.redis.expire(id, expireSeconds);
   }
 
   setField(id, key, value) {
     this.redis.hset(id, key, value);
   }
 
-  del(id) {
+  async del(id) {
+    const bucket = await this.redis.hgetAsync(id, 'bucket');
     this.redis.del(id);
-    return this.storage.del(id);
+    this.storage[bucket].del(id);
   }
 
   async ping() {
     await this.redis.pingAsync();
-    await this.storage.ping();
+    for (const bucket of this.storage) {
+      bucket.ping();
+    }
   }
 
   async metadata(id) {

server/storage/s3.js

@@ -2,8 +2,8 @@ const AWS = require('aws-sdk');
 const s3 = new AWS.S3();
 
 class S3Storage {
-  constructor(config, log) {
-    this.bucket = config.s3_bucket;
+  constructor(config, index, log) {
+    this.bucket = config.s3_buckets[index];
     this.log = log;
   }