Simple POC to use local Queues and local NoSQL Database

Technologies

• express
• cors
• lmdb
• better-queue

Code

I've added comment in the code to explain.

const express = require("express");
const app = express();
const cors = require("cors");
const { open } = require("lmdb");
const Queue = require("better-queue");

// This contains the local you want to do
// It has been removed in this case
// I've been using it along with puppeteer
// I need to batch process some crawling tasks.
const processing = require("./processing");

// Express Configurations
const hostname = "127.0.0.1";
const port = 3000;

// Process the content base on the input received
// In this POC I'm using a URL (it is a crawler)
// Then save the data returned by processing to the local NoSQL Database
// The data must be in JSON Format.
// After that a callback is returned to the queue system.
// data.id is used if defined otherwise the primary key is the current datetime
async function doStuff(input, cb) {
  const data = await processing(input);

  await myDB.put(data.id || new Date().toISOString(), { ...data });
  return cb(null, input);
}

// Create the Queue configuration and what function to use for processing
// The options are self explaning
const q = new Queue(doStuff, {
  concurrent: 2,
  maxRetries: 3,
  retryDelay: 5000,
  maxTimeout: 60000,
});

// Open the local NoSQL Database connection
// The directory used is named 'entries'
let myDB = open({
  path: "entries",
  // any options go here, we can turn on compression like this:
  compression: true,
});

// Express stuff
// Cors and req.body
app.use(cors());
app.use(express.json());

// Return statistics about the queue system
app.get("/status", async (req, res) => {
  res.send({
    queueInfo: q.getStats(),
    info: null,
  });
});

// Process urls pass in the req.body
// the structure is as follow:
// body: { datas: [String]}
app.post("/", async (req, res) => {
  try {
    // Handle Options Calls
    res.setHeader("Access-Control-Allow-Origin", "*");
    res.setHeader("Access-Control-Request-Method", "*");
    res.setHeader("Access-Control-Allow-Methods", "OPTIONS, GET");
    res.setHeader("Access-Control-Allow-Headers", "*");
    if (req.method === "OPTIONS") {
      res.writeHead(200);
      res.end();
      return;
    }

    // Extract Urls from the request
    const urls = req.body.datas;
    if (urls && urls.length === 0) throw new Error("No Url provided.");

    // Send each url to the queue
    // It will be processed using the configuration defined above.
    // The client does not have to wait for the full processing.
    // So currently there is no way to let know the user about the status of the request
    // Implementing a job id and returning it to the user is a good solution to track the status
    // It requires more work and in this case it wasn't required.
    urls.map((url) =>
      q
        .push(url)
        .on("finish", function (result) {
          console.log("Finished", result);
        })
        .on("failed", function (err) {
          console.error("Error:", err);
        })
    );

    res.send("Thank You!");
  } catch (e) {
    console.error(e.message);
    console.error(e.stack);
    res.send("Oops");
  }
});

// Start Express Server
app.listen(port, () => {
  console.log(`Server running at http://${hostname}:${port}/`);
});

---

I hope it provides some guidances or ideas about using queue and nosql locally with NodeJS.