NodeJs Worker Custom

// Importing required modules and functions
import fs from 'fs';
import { join } from 'path';
import { exit } from 'process';
import { Worker, isMainThread, workerData } from 'worker_threads';
import { upload } from './upsert';

// Define the directory path for the files
const directory = join(__dirname, '..', process.env.FOLDER_NAME || 'data');

// Read the file names in the directory
const fileNames = fs.readdirSync(directory);

// Define the number of threads to use, defaulting to 4
const THREAD_COUNT = process.env.THREAD_COUNT ? +process.env.THREAD_COUNT : 4;

// The main function that performs the file processing
function main() {
    // Check if there are any files to process
    if (fileNames.length < 0) {
        console.log('파일이 없습니다. 업로드를 종료합니다.'); // Print a message if no files are found and exit
        exit(1);
    }

    // Iterate over each file in the directory
    for (const dir of fileNames) {
        if (isMainThread) {
            const threads = new Set<Worker>();
            
            // Read the content of the current file
            const content = fs.readFileSync(join(__dirname, '..', process.env.FOLDER_NAME || 'data', dir), 'utf-8');
            
            // Count the number of lines in the file
            const dataLength = content.split('\n').length;

            // Create worker threads for parallel processing
            for (let i = 0; i < THREAD_COUNT; i++) {
                const start = Math.floor(dataLength / THREAD_COUNT) * i;
                const end = Math.floor(dataLength / THREAD_COUNT) * (i + 1);

                // Create a new worker thread and provide it with necessary data
                threads.add(new Worker(__filename, {
                    workerData: { filename: dir, lines: content.split('\n').slice(start, end) },
                }));
            }

            // Handle events from worker threads
            for (const worker of threads) {
                worker.on('error', (err) => {
                    console.error(err);
                });
                worker.on('message', (message) => {
                    // Handle the message received from the worker if needed
                });
            }
        } else {
            // Invoke the upload function in the worker thread with the provided data
            upload(workerData.filename, workerData.lines);
        }
    }
}

// Call the main function to start the processing
main();