My dev and I have set up a vibe.d API for handling requests and async tasks for the app.

There's a few routes we use for long running tasks. One for downloading data from an external API (from CRMs) and one for loading in mongo data and scanning them for duplicates (basically iterating over a large data set).

Because the scan takes up to a minute in our tests, we noticed that other routes are waiting for this task to complete before they send a response. We eliminated all of the obvious issues like db connection locks and such, using a simple void index(..) { res.writeBodyJson(["success": true ]); }

I have a suspicion that .getResult; is blocking the thread. For some reason the download doesn't seem to block the scan, but it appears the scan often blocks other requests.

This is how it looks:

Download:

private void hubspot_getCollections(HTTPServerRequest req, HTTPServerResponse res, UserId userId) {
  import mongo = core.mongo;

  auto d = async({
    enforce(mongo.getDowloadState(userId).state == mongo.DownloadState.NotInProgress);

    string entityType = "contacts"; //FIXME: hardcoded

    void push(mongo.DownloadState state, long currNum) {
      pushMsg(userId, `{"e": "downloadUpdate", "m": { "crmEntity": "`~entityType~`", "status": "`~state.to!string~`", "count": `~currNum.to!string~` } }`);
    }

    long counter;

    hubspot.downloadCollections(
      entityType,
      userId,
      (long currNum){
        counter = currNum;
        mongo.updateDownloadHeartbeating(userId, counter);
        push(mongo.DownloadState.InProgress, counter);
      }
    );

    mongo.doneDownloadHeartbeating(userId);
    push(mongo.DownloadState.NotInProgress, counter);

    return true;
  });

  d.getResult;

  res.writeJsonBody(["success": true]);
}

Duplicate scan:

void triggerFindMatchesJob(HTTPServerRequest req, HTTPServerResponse res, UserId userId) {
  import core.mongo;
  import core.matchalgos;
  import core.matcher;
  import std.stdio;

  MatchSetId matchSetId = BsonObjectID.fromString(req.params["matchSetId"]);

  const string entityType = "contacts"; //FIXME: hardcoded

  // Read Dses from DB
  auto dses = async({
    pushMsg(userId, `{"e": "findMatchesStart", "m": { "matchSetId": "`~matchSetId.toString~`" } }`);
    return get(Collection.ForDedup, entityType, userId);
  });

  // Matching
  auto dm = async({
    MatchSetIdentified[] mi = core.mongo.getMatchSets(userId, entityType);

    FieldMatchSetting[] fields;

    foreach(ref m; mi) {
      //TODO: for better performance it is need add getMatchSet method into core.mongo
      if(m.id == matchSetId) {
        foreach(ref f; m.matchSet.fields)
          if(f.match)
            fields ~= f;

        break;
      }
    }

    auto pr = Progress(
      (ubyte pct){
        pushMsg(userId, `{"e": "findMatchesProgress", "m": { "matchSetId": "`~matchSetId.toString~`", "progress": `~pct.to!string~` } }`);
      }
    );

    pr.iterationsStep = 2; //TODO: set more appropriate value

    DupeMatch dm;
    dm.progressCallback = &pr.progressUpdate;

    dm.execute(dses.getResult, fields);

    return dm;
  });

  // Wait for end of matching
  async({
    dm.getResult; // just wait for match is done
    //destroy(dses);

    pushMsg(userId, `{"e": "findMatchesEnd", "m": { "matchSetId": "`~matchSetId.toString~`" } }`);
    logInfo("Finished computing stuff in worker task.");

    return 1;
  }).getResult;

  // Adding Dses bodies into reply
  auto matchDocs = dm.getResult.toMatchDocs;
  Json ret = Json.emptyObject;

  ret["total"] = Json(matchDocs.length);
  ret["matches"] = Json.emptyArray;

  foreach(const ref md; matchDocs) {
    ret["matches"] ~= Json.emptyObject;
    ret["matches"][$-1] = md.serializeToJson;

    DseIdentified[] fetchedDses = getDsesByIds(Collection.ForDedup, md.ids);

    ret["matches"][$-1]["dses"] = fetchedDses.serializeToJson;
  }

  res.writeJsonBody(ret);
}

Any help figuring out why we're having this issue would be huge. We've been fighting with this for a week.