C++ och dess framtid att programmera minnessäkert - Hur går utvecklingen?

/** ---------------------------------------------------------------------------
 * @brief Updates the pattern list for files in the cache using regex patterns and multithreading.
 *
 * This method processes a list of regex patterns and applies them to the files stored in the cache.
 * It generates a list of lines in each file where the patterns are found and stores the results
 * in the "file-linelist" cache table. The method uses multithreading to process files in parallel.
 *
 * @param vectorRegexPatterns A vector of pairs containing regex patterns and their names.
 *                            The vector must not be empty.
 * @param argumentsList Arguments for pattern processing (e.g., segment specification, max lines)
 * @param iThreadCount Number of threads to use (0 = auto-detect)
 * @return A pair containing:
 *         - `bool`: `true` if the operation was successful, `false` otherwise.
 *         - `std::string`: An empty string on success, or an error message on failure.
 *
 * @pre The `vectorRegexPatterns` must not be empty.
 * @post The "file-linelist" cache table is updated with the lines where the patterns are found.
 *
 * @note COMMAND_ListLinesWithPattern must be thread-safe.
 */
std::pair<bool, std::string> CDocument::FILE_UpdatePatternList(const std::vector<std::pair<boost::regex, std::string>>& vectorRegexPatterns, const gd::argument::shared::arguments& argumentsList, int iThreadCount)
{                                                                                                  assert(vectorRegexPatterns.empty() == false); // Ensure the regex pattern list is not empty
   using namespace gd::table;
   
   auto* ptableFile = CACHE_Get("file");                                      // Retrieve the "file" cache table
   auto* ptableLineList = CACHE_Get("file-linelist", true);                   // Ensure the "file-linelist" table is in cache
                                                                                                   assert(ptableFile != nullptr); assert(ptableLineList != nullptr);

   std::string_view stringSegment;
   if( argumentsList.exists("segment") == true ) { stringSegment = argumentsList["segment"].as_string_view(); } // Get the segment (code, comment, string) to search in

   uint64_t uMax = argumentsList["max"].as_uint64();                          // Get the maximum number of lines to be printed
   auto uFileCount = ptableFile->get_row_count();                             // Total number of files to process

   // ## Thread synchronization variables

   std::atomic<uint64_t> uAtomicFileIndex(0);                                 // Current file being processed
   std::atomic<uint64_t> uAtomicProcessedCount(0);                            // Count of processed files
   std::atomic<uint64_t> uAtomicTotalLines(0);                                // Total lines found across all threads
   std::mutex mutexProgress;                                                  // Mutex to protect progress updates
   std::mutex mutexLineList;                                                  // Mutex to protect ptableLineList access
   std::vector<std::string> vectorErrors;                                     // Collect errors from all threads
   std::mutex mutexErrors;                                                    // Mutex to protect access to vectorErrors

   // ## Worker function to process files in parallel .........................
   auto process_ = [&](int iThreadId) -> void
   {
      // Create thread-local table for collecting results
      detail::columns* pcolumnsThread = new detail::columns{};
      ptableLineList->to_columns( *pcolumnsThread );
      std::unique_ptr<table> ptableLineListLocal = std::make_unique<table>(pcolumnsThread, 10, ptableLineList->get_flags(), 10); // Create local table with 10 rows pre-allocated
      
      while( true )
      {
         uint64_t uRowIndex = uAtomicFileIndex.fetch_add(1);                  // get thread safe current index and increment it
         if(uRowIndex >= uFileCount) { break; }
         
         try
         {
            // STEP 1: Get file info (ptableFile is read-only so no mutex needed)
            auto stringFolder = ptableFile->cell_get_variant_view(uRowIndex, "folder").as_string();
            auto stringFilename = ptableFile->cell_get_variant_view(uRowIndex, "filename").as_string();

            // STEP 2: Build full file path
            gd::file::path pathFile(stringFolder);
            pathFile += stringFilename;
            std::string stringFile = pathFile.string();

            auto uKey = ptableFile->cell_get_variant_view(uRowIndex, "key").as_uint64();

            // STEP 3: Find lines with regex patterns 
            gd::argument::shared::arguments arguments_({{"source", stringFile}, {"file-key", uKey}});
            if( stringSegment.empty() == false ) arguments_.set("segment", stringSegment.data()); // Set the segment (code, comment, string) to search in
            
            auto result_ = COMMAND_ListLinesWithPattern(arguments_, vectorRegexPatterns, ptableLineListLocal.get()); // Find lines with regex patterns and update the local table

            if(result_.first == false)
            {
               std::lock_guard<std::mutex> lockErrors(mutexErrors);
               vectorErrors.push_back("File: " + stringFile + " - " + result_.second);

               uint64_t uProcessed = uAtomicProcessedCount.fetch_add(1) + 1;   // Update progress even on failure
               if(uProcessed % 10 == 0)
               {
                  std::lock_guard<std::mutex> lockProgress(mutexProgress);
                  uint64_t uPercent = (uProcessed * 100) / uFileCount;
                  MESSAGE_Progress("", {{"percent", uPercent}, {"label", "Find in files"}, {"sticky", true}});
               }
               continue;                                                       // Skip to next file on error
            }

            // STEP 4: Append results to main table (FAST operation - mutex needed for thread safety)
            {
               std::lock_guard<std::mutex> lockLineList(mutexLineList);
               ptableLineList->append(ptableLineListLocal.get());              // Append the results from the local table to the main table

               // Update total line count and check if we've exceeded the maximum
               uint64_t uCurrentLines = uAtomicTotalLines.fetch_add(ptableLineListLocal->get_row_count()) + ptableLineListLocal->get_row_count();
               if( uCurrentLines > uMax ) 
               {
                  uAtomicFileIndex.store(uFileCount);                          // Signal other threads to stop by setting file index to max
               }
            }
            
            ptableLineListLocal->row_clear();                                  // Clear local table rows for next iteration

            uint64_t uProcessed = uAtomicProcessedCount.fetch_add(1) + 1;      // Update progress (thread-safe)
            if(uProcessed % 10 == 0)                                           // Show progress every 10 files
            {
               std::lock_guard<std::mutex> lockProgress(mutexProgress);
               uint64_t uPercent = (uProcessed * 100) / uFileCount;
               MESSAGE_Progress("", {{"percent", uPercent}, {"label", "Find in files"}, {"sticky", true}});
            }
         }
         catch(const std::exception& exception_)
         {
            std::lock_guard<std::mutex> lockErrors(mutexErrors);
            vectorErrors.push_back(std::string("Thread ") + std::to_string(iThreadId) + " error: " + exception_.what());
         }      
      }
   };

   // ## Prepare and run threads ..............................................

   if( iThreadCount <= 0 ) { iThreadCount = std::thread::hardware_concurrency(); } // Use hardware concurrency if no thread count is specified
   if(iThreadCount <= 0) { iThreadCount = 1; }                                 // Fallback to single thread if hardware_concurrency returns 0
   if( iThreadCount > 8 ) { iThreadCount = 8; }                                // Limit to 8 threads for performance and resource management
   if( ptableFile->size() < iThreadCount ) { iThreadCount = (int)ptableFile->size(); } // Limit threads to number of files

   // Create and launch worker threads
   std::vector<std::thread> vectorPatternThread;
   vectorPatternThread.reserve(iThreadCount);

   for(int i = 0; i < iThreadCount; ++i) { vectorPatternThread.emplace_back(process_, i); }

   // Wait for all threads to complete
   for(auto& threadWorker : vectorPatternThread) { threadWorker.join(); }

   MESSAGE_Progress("", {{"clear", true}});                                   // Clear progress message

   // ### Handle any collected errors
   if(!vectorErrors.empty())
   {
      for(const auto& stringError : vectorErrors) { ERROR_Add(stringError); }
   }

   return {true, ""};
}

/** ---------------------------------------------------------------------------
 * @brief Updates the pattern list for files in the cache using regex patterns and multithreading.
 *
 * This method processes a list of regex patterns and applies them to the files stored in the cache.
 * It generates a list of lines in each file where the patterns are found and stores the results
 * in the "file-linelist" cache table. The method uses multithreading to process files in parallel.
 *
 * @param vectorRegexPatterns A vector of pairs containing regex patterns and their names.
 *                            The vector must not be empty.
 * @param argumentsList Arguments for pattern processing (e.g., segment specification, max lines)
 * @param iThreadCount Number of threads to use (0 = auto-detect)
 * @return A pair containing:
 *         - `bool`: `true` if the operation was successful, `false` otherwise.
 *         - `std::string`: An empty string on success, or an error message on failure.
 *
 * @pre The `vectorRegexPatterns` must not be empty.
 * @post The "file-linelist" cache table is updated with the lines where the patterns are found.
 *
 * @note COMMAND_ListLinesWithPattern must be thread-safe.
 */
std::pair<bool, std::string> CDocument::FILE_UpdatePatternList(const std::vector<std::pair<boost::regex, std::string>>& vectorRegexPatterns, const gd::argument::shared::arguments& argumentsList, int iThreadCount)
{                                                                                                  assert(vectorRegexPatterns.empty() == false); // Ensure the regex pattern list is not empty
   using namespace gd::table;
   
   auto* ptableFile = CACHE_Get("file");                                      // Retrieve the "file" cache table
   auto* ptableLineList = CACHE_Get("file-linelist", true);                   // Ensure the "file-linelist" table is in cache
                                                                                                   assert(ptableFile != nullptr); assert(ptableLineList != nullptr);

   std::string_view stringSegment;
   if( argumentsList.exists("segment") == true ) { stringSegment = argumentsList["segment"].as_string_view(); } // Get the segment (code, comment, string) to search in

   uint64_t uMax = argumentsList["max"].as_uint64();                          // Get the maximum number of lines to be printed
   auto uFileCount = ptableFile->get_row_count();                             // Total number of files to process

   // ## Thread synchronization variables

   std::atomic<uint64_t> uAtomicFileIndex(0);                                 // Current file being processed
   std::atomic<uint64_t> uAtomicProcessedCount(0);                            // Count of processed files
   std::atomic<uint64_t> uAtomicTotalLines(0);                                // Total lines found across all threads
   std::mutex mutexProgress;                                                  // Mutex to protect progress updates
   std::mutex mutexLineList;                                                  // Mutex to protect ptableLineList access
   std::vector<std::string> vectorErrors;                                     // Collect errors from all threads
   std::mutex mutexErrors;                                                    // Mutex to protect access to vectorErrors

   // ## Worker function to process files in parallel .........................
   auto process_ = [&](int iThreadId) -> void
   {
      // Create thread-local table for collecting results
      detail::columns* pcolumnsThread = new detail::columns{};
      ptableLineList->to_columns( *pcolumnsThread );
      std::unique_ptr<table> ptableLineListLocal = std::make_unique<table>(pcolumnsThread, 10, ptableLineList->get_flags(), 10); // Create local table with 10 rows pre-allocated
      
      while( true )
      {
         uint64_t uRowIndex = uAtomicFileIndex.fetch_add(1);                  // get thread safe current index and increment it
         if(uRowIndex >= uFileCount) { break; }
         
         try
         {
            // STEP 1: Get file info (ptableFile is read-only so no mutex needed)
            auto stringFolder = ptableFile->cell_get_variant_view(uRowIndex, "folder").as_string();
            auto stringFilename = ptableFile->cell_get_variant_view(uRowIndex, "filename").as_string();

            // STEP 2: Build full file path
            gd::file::path pathFile(stringFolder);
            pathFile += stringFilename;
            std::string stringFile = pathFile.string();

            auto uKey = ptableFile->cell_get_variant_view(uRowIndex, "key").as_uint64();

            // STEP 3: Find lines with regex patterns 
            gd::argument::shared::arguments arguments_({{"source", stringFile}, {"file-key", uKey}});
            if( stringSegment.empty() == false ) arguments_.set("segment", stringSegment.data()); // Set the segment (code, comment, string) to search in
            
            auto result_ = COMMAND_ListLinesWithPattern(arguments_, vectorRegexPatterns, ptableLineListLocal.get()); // Find lines with regex patterns and update the local table

            if(result_.first == false)
            {
               std::lock_guard<std::mutex> lockErrors(mutexErrors);
               vectorErrors.push_back("File: " + stringFile + " - " + result_.second);

               uint64_t uProcessed = uAtomicProcessedCount.fetch_add(1) + 1;   // Update progress even on failure
               if(uProcessed % 10 == 0)
               {
                  std::lock_guard<std::mutex> lockProgress(mutexProgress);
                  uint64_t uPercent = (uProcessed * 100) / uFileCount;
                  MESSAGE_Progress("", {{"percent", uPercent}, {"label", "Find in files"}, {"sticky", true}});
               }
               continue;                                                       // Skip to next file on error
            }

            // STEP 4: Append results to main table (FAST operation - mutex needed for thread safety)
            {
               std::lock_guard<std::mutex> lockLineList(mutexLineList);
               ptableLineList->append(ptableLineListLocal.get());              // Append the results from the local table to the main table

               // Update total line count and check if we've exceeded the maximum
               uint64_t uCurrentLines = uAtomicTotalLines.fetch_add(ptableLineListLocal->get_row_count()) + ptableLineListLocal->get_row_count();
               if( uCurrentLines > uMax ) 
               {
                  uAtomicFileIndex.store(uFileCount);                          // Signal other threads to stop by setting file index to max
               }
            }
            
            ptableLineListLocal->row_clear();                                  // Clear local table rows for next iteration

            uint64_t uProcessed = uAtomicProcessedCount.fetch_add(1) + 1;      // Update progress (thread-safe)
            if(uProcessed % 10 == 0)                                           // Show progress every 10 files
            {
               std::lock_guard<std::mutex> lockProgress(mutexProgress);
               uint64_t uPercent = (uProcessed * 100) / uFileCount;
               MESSAGE_Progress("", {{"percent", uPercent}, {"label", "Find in files"}, {"sticky", true}});
            }
         }
         catch(const std::exception& exception_)
         {
            std::lock_guard<std::mutex> lockErrors(mutexErrors);
            vectorErrors.push_back(std::string("Thread ") + std::to_string(iThreadId) + " error: " + exception_.what());
         }      
      }
   };

   // ## Prepare and run threads ..............................................

   if( iThreadCount <= 0 ) { iThreadCount = std::thread::hardware_concurrency(); } // Use hardware concurrency if no thread count is specified
   if(iThreadCount <= 0) { iThreadCount = 1; }                                 // Fallback to single thread if hardware_concurrency returns 0
   if( iThreadCount > 8 ) { iThreadCount = 8; }                                // Limit to 8 threads for performance and resource management
   if( ptableFile->size() < iThreadCount ) { iThreadCount = (int)ptableFile->size(); } // Limit threads to number of files

   // Create and launch worker threads
   std::vector<std::thread> vectorPatternThread;
   vectorPatternThread.reserve(iThreadCount);

   for(int i = 0; i < iThreadCount; ++i) { vectorPatternThread.emplace_back(process_, i); }

   // Wait for all threads to complete
   for(auto& threadWorker : vectorPatternThread) { threadWorker.join(); }

   MESSAGE_Progress("", {{"clear", true}});                                   // Clear progress message

   // ### Handle any collected errors
   if(!vectorErrors.empty())
   {
      for(const auto& stringError : vectorErrors) { ERROR_Add(stringError); }
   }

   return {true, ""};
}