استكشاف المتن النصي

import csv

with open("slm-corpus.csv", newline="") as f:
    reader = csv.DictReader(f)
    rows = list(reader)

print(f"Rows:    {len(rows)}")
print(f"Columns: {list(rows[0].keys())}")

total_chars = sum(len(row["text"]) for row in rows)
avg_len = total_chars / len(rows) if rows else 0

print(f"Total characters: {total_chars:,}")
print(f"Average row length: {avg_len:.0f} characters")

for i, row in enumerate(rows[:5]):
    preview = row["text"][:150].replace("\n", " ")
    print(f"[{i}] {preview}...")

unique_texts = set(row["text"] for row in rows)
print(f"Unique rows: {len(unique_texts)} / {len(rows)}")

if len(unique_texts) < len(rows):
    print(f"Warning: {len(rows) - len(unique_texts)} duplicate rows found")

short_rows = [row for row in rows if len(row["text"].split()) < 3]
print(f"Rows with fewer than 3 words: {len(short_rows)}")

def corpus_summary(path):
    import csv
    with open(path, newline="") as f:
        reader = csv.DictReader(f)
        rows = list(reader)

    texts = [row["text"] for row in rows]
    total_chars = sum(len(t) for t in texts)
    unique = len(set(texts))

    print(f"Rows: {len(rows)}")
    print(f"Unique: {unique}")
    print(f"Total chars: {total_chars:,}")
    print(f"Avg length: {total_chars / len(rows):.0f}")
    print(f"Columns: {list(rows[0].keys())}")

def corpus_quality(path):
    import csv
    with open(path, newline="") as f:
        reader = csv.DictReader(f)
        rows = list(reader)

    texts = [row["text"] for row in rows]
    lengths = [len(t.split()) for t in texts]

    return {
        "rows": len(rows),
        "unique": len(set(texts)),
        "total_chars": sum(len(t) for t in texts),
        "avg_length": sum(lengths) / len(lengths) if lengths else 0,
        "min_length": min(lengths) if lengths else 0,
        "max_length": max(lengths) if lengths else 0,
    }