TaskGraphRuntime.hpp

Go to the documentation of this file.

// NIST-developed software is provided by NIST as a public service. You may use, copy and distribute copies of the software in any medium, provided that you keep intact this entire notice. You may improve, modify and create derivative works of the software or any portion of the software, and you may copy and distribute such modifications or works. Modified works should carry a notice stating that you changed the software and should note the date and nature of any such change. Please explicitly acknowledge the National Institute of Standards and Technology as the source of the software.
// NIST-developed software is expressly provided "AS IS." NIST MAKES NO WARRANTY OF ANY KIND, EXPRESS, IMPLIED, IN FACT OR ARISING BY OPERATION OF LAW, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT AND DATA ACCURACY. NIST NEITHER REPRESENTS NOR WARRANTS THAT THE OPERATION OF THE SOFTWARE WILL BE UNINTERRUPTED OR ERROR-FREE, OR THAT ANY DEFECTS WILL BE CORRECTED. NIST DOES NOT WARRANT OR MAKE ANY REPRESENTATIONS REGARDING THE USE OF THE SOFTWARE OR THE RESULTS THEREOF, INCLUDING BUT NOT LIMITED TO THE CORRECTNESS, ACCURACY, RELIABILITY, OR USEFULNESS OF THE SOFTWARE.
// You are solely responsible for determining the appropriateness of using and distributing the software and you assume all risks associated with its use, including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and the unavailability or interruption of operation. This software is not intended to be used in any situation where a failure could cause risk of injury or damage to property. The software developed by NIST employees is not subject to copyright protection within the United States.

#ifndef HTGS_TASKGRAPHRUNTIME_HPP
#define HTGS_TASKGRAPHRUNTIME_HPP

#include <thread>
#include <htgs/api/TaskGraphConf.hpp>
#include <htgs/core/task/AnyTaskManager.hpp>

namespace htgs {
class TaskGraphRuntime {

 public:

  TaskGraphRuntime(AnyTaskGraphConf *graph) {
    this->graph = graph;
    this->executed = false;
#if defined (USE_NVTX) && defined (USE_MINIMAL_NVTX)
    domainInitialize = nvtxDomainCreateA("Initialize");
    domainExecute = nvtxDomainCreateA("Execute");
    domainWait = nvtxDomainCreateA("Wait");
    domainWaitForMem = nvtxDomainCreateA("Wait for Memory");
    domainReleaseMem = nvtxDomainCreateA("Release Memory");
    domainShutdown = nvtxDomainCreateA("Shutdown");
#elif defined (USE_NVTX)
    domainInitialize = nullptr;
    domainExecute = nullptr;
    domainWait = nullptr;
    domainWaitForMem = nullptr;
    domainReleaseMem = nullptr;
    domainShutdown = nullptr;
#endif
  }

  ~TaskGraphRuntime() {
    for (TaskManagerThread *t : runtimeThreads) {
      if (t) {
        HTGS_DEBUG_VERBOSE("TaskGraphRuntime: " << this << " Freeing memory for TaskManagerThread: " << t);

        delete t;
        t = nullptr;
      }
    }

    for (std::thread *t : threads) {
      if (t) {


        HTGS_ASSERT(!t->joinable(), "Trying to delete thread that has not been joined. You must call 'waitForRuntime' prior to deleting the TaskGraphRuntime. (see https://pages.nist.gov/HTGS/doxygen/classhtgs_1_1_task_graph_runtime.html#a8f2eaf040695178b6f61db7b0ee16c89)");


        delete t;
        t = nullptr;
      }
    }

    if (graph) {
      HTGS_DEBUG_VERBOSE("TaskGraphRuntime: " << this << " Freeing memory for graph: " << graph);

      delete graph;
      graph = nullptr;
    }
#if defined (USE_NVTX) && (USE_MINIMAL_NVTX)
    nvtxDomainDestroy(domainInitialize);
    nvtxDomainDestroy(domainExecute);
    nvtxDomainDestroy(domainWait);
    nvtxDomainDestroy(domainWaitForMem);
    nvtxDomainDestroy(domainReleaseMem);
    nvtxDomainDestroy(domainShutdown);
#endif
  }

  void waitForRuntime() {
    for (std::thread *t : threads) {
      if (t->joinable())
        t->join();
    }

    this->graph->shutdown();
  }

  void executeAndWaitForRuntime() {
    executeRuntime();
    waitForRuntime();
  }

  void terminateAll() {
    for (TaskManagerThread *t : runtimeThreads) {
      t->terminate();
    }
  }

  void executeRuntime() {
    if (executed)
      return;

    // Initialize graph and setup task graph taskGraphCommunicator
    this->graph->initialize();

    std::list<AnyTaskManager *> *vertices = this->graph->getTaskManagers();
    std::list<AnyTaskManager *> newVertices;
    HTGS_DEBUG_VERBOSE("Launching runtime for " << vertices->size() << " vertices");


    for (AnyTaskManager *task : *vertices) {

      size_t numThreads = task->getNumThreads();

      HTGS_DEBUG_VERBOSE("Spawning " << numThreads << " threads for task " << task->getName());

      if (numThreads > 0) {
        std::list<AnyTaskManager *> taskList;
        std::shared_ptr<std::atomic_size_t>
            atomicNumThreads = std::shared_ptr<std::atomic_size_t>(new std::atomic_size_t(numThreads));
        taskList.push_back(task);

        for (size_t i = 1; i < numThreads; i++) {
          AnyTaskManager *taskCopy = task->copy(true);

#ifdef WS_PROFILE
          // Generate . . . and send data . . .
          std::shared_ptr<ProfileData> producerData(new CreateNodeProfile(taskCopy->getTaskFunction(), graph, taskCopy->getName()));
          graph->sendProfileData(producerData);

          std::shared_ptr<ProfileData> connectorConsumerData(new CreateEdgeProfile(taskCopy->getInputConnector().get(), taskCopy->getTaskFunction(), "", nullptr));
          std::shared_ptr<ProfileData> producerConnectorData(new CreateEdgeProfile(taskCopy->getTaskFunction(), taskCopy->getOutputConnector().get(), "", nullptr));

          graph->sendProfileData(connectorConsumerData);
          graph->sendProfileData(producerConnectorData);
#endif

          // Add communicator to task copy to enable communication
          // TODO: Delete or Add #ifdef
//          taskCopy->setTaskGraphCommunicator(graph->getTaskGraphCommunicator());
          taskList.push_back(taskCopy);
          newVertices.push_back(taskCopy);
        }
        size_t threadId = 0;

#if defined (USE_NVTX) && defined (USE_MINIMAL_NVTX)
        nvtxDomainHandle_t taskDomain = nullptr;
#elif defined (USE_NVTX)
        nvtxDomainHandle_t taskDomain = nvtxDomainCreateA(task->getName().c_str());
#endif

        for (AnyTaskManager *taskItem : taskList) {
#if defined (USE_NVTX) && defined (USE_MINIMAL_NVTX)
          NVTXProfiler *profiler = new NVTXProfiler(std::to_string(threadId) + ":" + taskItem->getName(), taskDomain, domainInitialize, domainExecute, domainWait, domainWaitForMem, domainReleaseMem, domainShutdown);
          taskItem->setProfiler(profiler);
#elif defined (USE_NVTX)
          NVTXProfiler *profiler = new NVTXProfiler(std::to_string(threadId), taskDomain, domainInitialize, domainExecute, domainWait, domainWaitForMem, domainReleaseMem, domainShutdown);
          taskItem->setProfiler(profiler);
#endif



          TaskManagerThread *runtimeThread = new TaskManagerThread(threadId, taskItem, atomicNumThreads, graph->getInitializationCondition(), graph->getInitializationMutex());
          std::thread *thread = new std::thread(&TaskManagerThread::run, runtimeThread);
          this->threads.push_back(thread);
          runtimeThreads.push_back(runtimeThread);
          threadId++;
        }

      } else {
        std::cerr << task->getName() << " has no threads specified." << std::endl;
      }
    }


#ifdef WS_PROFILE
    std::shared_ptr<ProfileData> graphCreationComplete(new GraphCompleteProfile(graph));
    graph->sendProfileData(graphCreationComplete);
#endif

    for (AnyTaskManager *newVertex : newVertices) {
      graph->addTaskManager(newVertex);
    }


    this->executed = true;

    graph->finishedSetup();
  }

 private:
  std::list<std::thread *> threads; 
  AnyTaskGraphConf *graph; 
  std::list<TaskManagerThread *> runtimeThreads; 
  bool executed; 

#ifdef USE_NVTX
  nvtxDomainHandle_t domainInitialize;
  nvtxDomainHandle_t domainExecute;
  nvtxDomainHandle_t domainWait;
  nvtxDomainHandle_t domainWaitForMem;
  nvtxDomainHandle_t domainReleaseMem;
  nvtxDomainHandle_t domainShutdown;
#endif

};
}

#endif //HTGS_TASKGRAPHRUNTIME_HPP