高效處理大型 CSV 檔案是許多應用程式中的常見要求，從資料分析到 ETL（提取、轉換、載入）過程。在本文中，我想對四種流行程式語言（Golang、帶有 NestJS 的 NodeJS、PHP 和 Python）在 MacBook Pro M1 上處理大型 CSV 檔案的效能進行基準測試。我的目標是確定哪種語言可以為該任務提供最佳效能。
硬體：MacBook Pro M1，256GB SSD，8GB RAM
軟體：
我使用了一個名為 sales_data.csv 的合成 CSV 文件，其中包含大約 100 萬行，每行都包含交易詳細信息，例如 transaction_id、product_id、數量、價格和時間戳。
對於每種語言，腳本執行以下任務：
以下是每種語言使用的腳本：
sales.go
package main
import (
"encoding/csv"
"fmt"
"os"
"strconv"
"time"
)
func main() {
start := time.Now()
file, err := os.Open("../generate-csv/sales_data.csv")
if err != nil {
fmt.Println("Error:", err)
return
}
defer file.Close()
reader := csv.NewReader(file)
_, _ = reader.Read() // Skip header
totalSales := 0.0
productSales := make(map[string]float64)
for {
line, err := reader.Read()
if err != nil {
break
}
productID := line[1]
quantity, _ := strconv.Atoi(line[2])
price, _ := strconv.ParseFloat(line[3], 64)
total := float64(quantity) * price
totalSales += total
productSales[productID] += total
}
var topProduct string
var topSales float64
for product, sales := range productSales {
if sales > topSales {
topProduct = product
topSales = sales
}
}
elapsed := time.Since(start)
fmt.Printf("Golang Execution time: %s\n", elapsed)
fmt.Printf("Total Sales: $%.2f\n", totalSales)
fmt.Printf("Top Product: %s with sales $%.2f\n", topProduct, topSales)
}
csv.service.ts
import { Injectable } from '@nestjs/common';
import * as fs from 'fs';
import * as fastcsv from 'fast-csv';
// path file CSV
const GLOBAL_CSV_PATH = '../generate-csv/sales_data.csv';
@Injectable()
@Injectable()
export class CsvService {
async parseCsv(): Promise<{
nestExecutionTime: number;
totalSales: number;
topProductSales: number;
}> {
return new Promise((resolve, reject) => {
const startTime = process.hrtime();
let totalSales = 0;
const productSales: { [key: string]: number } = {};
fs.createReadStream(GLOBAL_CSV_PATH)
.pipe(fastcsv.parse({ headers: true, delimiter: ',' }))
.on('data', (row) => {
const productID = row.product_id;
const quantity = parseInt(row.quantity, 10);
const price = parseFloat(row.price);
const total = quantity * price;
totalSales += total;
if (!productSales[productID]) {
productSales[productID] = 0;
}
productSales[productID] += total;
})
.on('end', () => {
const topProduct = Object.keys(productSales).reduce((a, b) =>
productSales[a] > productSales[b] ? a : b,
);
const topProductSales = productSales[topProduct] || 0;
const endTime = process.hrtime(startTime);
const nestExecutionTime = endTime[0] + endTime[1] / 1e9;
console.log(`NestJS Execution time: ${nestExecutionTime} seconds`);
console.log(`Total Sales: $${totalSales}`);
console.log(
`Top Product: ${topProduct} with sales $${topProductSales}`,
);
resolve({
nestExecutionTime,
totalSales,
topProductSales,
});
})
.on('error', (error) => reject(error));
});
}
}
csv.controller.ts
import { Controller, Get } from '@nestjs/common';
import { CsvService } from './csv.service';
@Controller('csv')
export class CsvController {
constructor(private readonly csvService: CsvService) {}
@Get('parse')
async parseCsv(): Promise<{
nestExecutionTime: number;
totalSales: number;
topProductSales: number;
}> {
return this.csvService.parseCsv();
}
}
sales.php
<?php
$start_time = microtime(true);
$file = fopen("../generate-csv/sales_data.csv", "r");
$total_sales = 0;
$product_sales = [];
fgetcsv($file); // Skip header
while (($line = fgetcsv($file)) !== false) {
$product_id = $line[1];
$quantity = (int)$line[2];
$price = (float)$line[3];
$total = $quantity * $price;
$total_sales += $total;
if (!isset($product_sales[$product_id])) {
$product_sales[$product_id] = 0;
}
$product_sales[$product_id] += $total;
}
fclose($file);
arsort($product_sales);
$top_product = array_key_first($product_sales);
$end_time = microtime(true);
$execution_time = ($end_time - $start_time);
echo "PHP Execution time: ".$execution_time." seconds\n";
echo "Total Sales: $".$total_sales."\n";
echo "Top Product: ".$top_product." with sales $".$product_sales[$top_product]."\n";
import csv
import time
# Input file name config
input_file = '../generate-csv/sales_data.csv'
def parse_csv(file_path):
start_time = time.time()
total_sales = 0
product_sales = {}
with open(file_path, mode='r') as file:
reader = csv.DictReader(file)
for row in reader:
product_id = row['product_id']
quantity = int(row['quantity'])
price = float(row['price'])
total = quantity * price
total_sales += total
if product_id not in product_sales:
product_sales[product_id] = 0
product_sales[product_id] += total
top_product = max(product_sales, key=product_sales.get)
execution_time = time.time() - start_time
return {
'total_sales': total_sales,
'top_product': top_product,
'top_product_sales': product_sales[top_product],
'execution_time': execution_time,
}
if __name__ == "__main__":
result = parse_csv(input_file)
print(f"Python Execution time: {result['execution_time']:.2f} seconds")
print(f"Total Sales: ${result['total_sales']:.2f}")
print(f"Top Product: {result['top_product']} with sales ${
result['top_product_sales']:.2f}")
以下是我們的基準測試結果：
我的基準測試揭示了一些有趣的見解：
執行時間：Golang 在執行時間方面表現最好，PHP8 緊隨其後，而 NestJS 完成任務的時間最長。
記憶體使用：Build NestJS 表現出高效的記憶體使用，而 Python 表現出更高的記憶體消耗。
易於實現：Golang 提供了最簡單的實現，而 NestJS 需要更多的程式碼行和複雜性。
根據我的發現，Golang 提供了最佳的效能速度和記憶體效率，使其成為處理大型資料集的絕佳選擇。
您可以在我的 Github 儲存庫上取得完整程式碼
csv-解析-戰鬥。
以上是CSV 檔案處理基準測試：Golang、NestJS、PHP、Python的詳細內容。更多資訊請關注PHP中文網其他相關文章！
