Customer 360° Data Integration Challenge

import pandas as pd
import numpy as np
from fuzzywuzzy import fuzz
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import precision_recall_fscore_support

# Sample function to generate matching features
def generate_matching_features(record1, record2):
    """Generate features for matching two customer records across systems"""
    features = {}
    
    # Name similarity features
    name1 = record1.get('FULL_NAME', record1.get('display_name', f"{record1.get('First_Name', '')} {record1.get('Last_Name', '')}"))
    name2 = record2.get('FULL_NAME', record2.get('display_name', f"{record2.get('First_Name', '')} {record2.get('Last_Name', '')}"))
    
    features['name_token_set_ratio'] = fuzz.token_set_ratio(name1, name2)
    features['name_token_sort_ratio'] = fuzz.token_sort_ratio(name1, name2)
    
    # Phone similarity
    phone1 = str(record1.get('PHONE', record1.get('contact', {}).get('phone', '')))
    phone2 = str(record2.get('PHONE', record2.get('contact', {}).get('phone', '')))
    
    # Normalize phone numbers - remove non-digits and compare last 10 digits
    phone1_digits = ''.join([c for c in phone1 if c.isdigit()])[-10:]
    phone2_digits = ''.join([c for c in phone2 if c.isdigit()])[-10:]
    
    features['phone_exact_match'] = int(phone1_digits == phone2_digits and len(phone1_digits) == 10)
    
    # Address similarity
    # Extract address components in a format-agnostic way
    addr1 = get_normalized_address(record1)
    addr2 = get_normalized_address(record2)
    
    features['address_similarity'] = fuzz.token_set_ratio(addr1, addr2)
    
    # Date of birth comparison
    dob1 = normalize_date(record1.get('DOB', record1.get('Birth_Date', record1.get('birth_date'))))
    dob2 = normalize_date(record2.get('DOB', record2.get('Birth_Date', record2.get('birth_date'))))
    
    features['dob_match'] = int(dob1 == dob2 and dob1 is not None)
    
    # Additional features for confidence calculation
    features['source_system_reliability'] = get_system_reliability_score(record1, record2)
    features['data_completeness'] = calculate_completeness_score(record1, record2)
    
    return features

# Train a matching model with labeled data
def train_matching_model(labeled_pairs):
    """Train a model to predict if two records match based on similarity features"""
    X = []
    y = []
    
    for pair, is_match in labeled_pairs:
        features = generate_matching_features(pair[0], pair[1])
        X.append(list(features.values()))
        y.append(is_match)
    
    model = RandomForestClassifier(n_estimators=100, max_depth=10)
    model.fit(X, y)
    
    return model

# Resolve entities across multiple systems
def resolve_customer_entities(customer_records_by_system):
    """Identify unique customers across system boundaries"""
    all_comparisons = []
    for sys1, records1 in customer_records_by_system.items():
        for sys2, records2 in customer_records_by_system.items():
            if sys1 >= sys2:  # Avoid duplicate comparisons
                continue
                
            for rec1 in records1:
                for rec2 in records2:
                    features = generate_matching_features(rec1, rec2)
                    all_comparisons.append((rec1, rec2, features))
    
    # Apply matching model
    match_predictions = matching_model.predict_proba([list(f.values()) for _, _, f in all_comparisons])
    
    # Create clusters of matching records
    customer_clusters = build_customer_clusters(all_comparisons, match_predictions)
    
    # Create golden records
    golden_records = []
    for cluster in customer_clusters:
        golden_record = create_golden_record(cluster)
        golden_records.append(golden_record)
    
    return golden_records