تطبيع أعداد أزواج الكلمات

import csv
import string
from collections import defaultdict

with open("slm-corpus.csv", newline="") as f:
    reader = csv.DictReader(f)
    texts = [row["text"] for row in reader]

def load_corpus(path):
    with open(path, newline="") as f:
        reader = csv.DictReader(f)
        return [row["text"] for row in reader]

def tokenize(text):
    text = text.lower()
    for char in string.punctuation:
        text = text.replace(char, " ")
    return text.split()

def build_bigrams(tokens):
    bigrams = defaultdict(lambda: defaultdict(int))
    for i in range(len(tokens) - 1):
        bigrams[tokens[i]][tokens[i + 1]] += 1
    return dict(bigrams)

def normalize_bigrams(bigrams):
    normalized = {}
    for word, followers in bigrams.items():
        total = sum(followers.values())
        normalized[word] = {w: c / total for w, c in followers.items()}
    return normalized

raw_bigrams = {"the": {"cat": 15, "dog": 5, "bird": 10}}
norm = normalize_bigrams(raw_bigrams)

print(norm["the"])
# {'cat': 0.5, 'dog': 0.1667, 'bird': 0.3333}

print(sum(norm["the"].values()))  # 1.0

import random

followers = list(norm["the"].keys())
weights = list(norm["the"].values())
next_word = random.choices(followers, weights=weights, k=1)[0]
print(f"Next word: {next_word}")

def normalize_bigrams(bigrams):
    normalized = {}
    for word, followers in bigrams.items():
        if not followers:
            continue  # skip words with no followers
        total = sum(followers.values())
        normalized[word] = {w: c / total for w, c in followers.items()}
    return normalized

texts = load_corpus("slm-corpus.csv")
tokens = tokenize(" ".join(texts))
bigrams = build_bigrams(tokens)
model = normalize_bigrams(bigrams)

# Check a sample
print(f"Words in model: {len(model)}")
print(f"Followers of 'the': {list(model.get('the', {}).keys())[:5]}")

import json

with open("bigram_model.json", "w") as f:
    json.dump(model, f)

# Reload later
with open("bigram_model.json") as f:
    model = json.load(f)

model = normalize_bigrams(bigrams)
the_followers = model.get("the", {})
top_follower = max(the_followers, key=the_followers.get)
print(f"Most likely after 'the': '{top_follower}' ({the_followers[top_follower]:.3f})")

def bigram_stats(model, top_n=10):
    words = sorted(model, key=lambda w: sum(model[w].values()), reverse=True)
    for word in words[:top_n]:
        followers = model[word]
        total = sum(followers.values())
        best = max(followers, key=followers.get)
        print(f"'{word}': {len(followers)} followers, best=''{best}'' ({followers[best]:.3f})")