NumPy & Pandas for High-Performance Analytics

import numpy as np

# 1. Direct conversion from Python sequences
arr_1d = np.array([10, 20, 30, 40], dtype=np.int32)
matrix_2d = np.array([[1.5, 2.5], [3.5, 4.5]], dtype=np.float64)

# 2. Structured range generators
grid_steps = np.arange(start=0, stop=100, step=10)       # Integers [0, 10, ..., 90]
continuous_samples = np.linspace(start=0.0, stop=1.0, num=5) # 5 equidistant floats

# 3. Memory pre-allocation functions
zeros_tensor = np.zeros(shape=(3, 4), dtype=np.float32)   # Filled with 0.0
ones_vector = np.ones(shape=(5,), dtype=np.int64)          # Filled with 1
identity_matrix = np.eye(N=4, dtype=np.float64)           # 4x4 Identity Matrix

import numpy as np

raw_matrix = np.array([
    [10, 20, 30],
    [40, 50, 60],
    [70, 80, 90]
])

# Slicing creates a VIEW
sub_view = raw_matrix[0:2, 1:3]
sub_view[0, 0] = 999  # Mutates raw_matrix[0, 1] to 999!

# Boolean Masking creates a COPY
condition_mask = raw_matrix > 50
filtered_values = raw_matrix[condition_mask]
filtered_values[0] = -1  # raw_matrix remains unchanged

import numpy as np

# Matrix A: Feature matrix (3 samples, 4 sensor features)
features = np.array([
    [12.0, 15.0, 110.0, 0.8],
    [14.0, 16.0, 115.0, 0.9],
    [11.0, 14.0, 105.0, 0.7]
])

# 1. Axis-based reduction: Compute mean per column (across rows: axis=0)
col_means = features.mean(axis=0)  # Shape: (4,)
col_stds = features.std(axis=0)    # Shape: (4,)

# 2. Broadcasting Z-score Standardization: (3, 4) - (4,) / (4,)
standardized_features = (features - col_means) / col_stds
print(standardized_features)

import numpy as np

# Initialize isolated Generator with fixed seed for deterministic reproducibility
rng = np.random.default_rng(seed=42)

# 1. Continuous Uniform Distribution in range [0.0, 1.0)
uniform_samples = rng.uniform(low=0.0, high=1.0, size=(1000,))

# 2. Gaussian / Normal Distribution: Mean = 100.0, Standard Deviation = 15.0
iq_scores = rng.normal(loc=100.0, scale=15.0, size=(5000,))

# 3. Discrete Uniform Integers: Simulate 6-sided dice rolls
dice_rolls = rng.integers(low=1, high=7, size=10)

# 4. Stochastic Permutation: Shuffle array in-place
dataset_indices = np.arange(10)
rng.shuffle(dataset_indices)

import pandas as pd

# 1. Ingesting CSV with explicit dtypes, date parsing, and index designation
df_csv = pd.read_csv(
    "enterprise_transactions.csv",
    sep=",",
    parse_dates=["transaction_date"],
    dtype={"customer_id": "int32", "status": "category"},
    na_values=["NA", "null", "MISSING"]
)

# 2. Ingesting Excel with specific sheet extraction
df_excel = pd.read_excel("financial_model.xlsx", sheet_name="Q3_Projections")

# 3. Ingesting semi-structured JSON records
df_json = pd.read_json("telemetry.json", orient="records")

import pandas as pd

# Preliminary Inspection
print(df_csv.info())         # Memory consumption, non-null counts, dtypes
print(df_csv.describe())     # 5-number summary (mean, std, quartiles)

# 1. Label-based selection via .loc
high_value_subset = df_csv.loc[
    0:100, 
    ["customer_id", "transaction_amount", "status"]
]

# 2. Integer position-based selection via .iloc
first_five_rows_first_three_cols = df_csv.iloc[0:5, 0:3]

# 3. Compound Boolean Filtering (Bitwise & for AND, | for OR)
filtered_df = df_csv[
    (df_csv["transaction_amount"] >= 500.0) &
    (df_csv["status"] == "completed")
]

# 1. Adding a vectorized computed column
df_csv["tax_amount"] = df_csv["transaction_amount"] * 0.18

# 2. Renaming columns cleanly
df_csv = df_csv.rename(columns={
    "customer_id": "client_identifier",
    "status": "order_state"
})

# 3. Dropping unneeded columns (axis=1)
df_clean = df_csv.drop(columns=["unneeded_legacy_field"])

import pandas as pd

# 1. Audit missing values per column
missing_summary = df_clean.isna().sum()

# 2. Strategy A: Drop rows where critical primary keys are null
df_clean = df_clean.dropna(subset=["client_identifier"])

# 3. Strategy B: Impute missing numerical metrics with median
median_tx = df_clean["transaction_amount"].median()
df_clean["transaction_amount"] = df_clean["transaction_amount"].fillna(median_tx)

# 4. Strategy C: Forward-fill / Backward-fill for time-series sequences
df_clean["sensor_reading"] = df_clean["sensor_reading"].ffill()

# 5. Deduplication: Identify and drop duplicate transactions
duplicate_count = df_clean.duplicated(subset=["client_identifier", "transaction_date"]).sum()
df_deduped = df_clean.drop_duplicates(
    subset=["client_identifier", "transaction_date"],
    keep="last"
)

import pandas as pd

# 1. Multi-metric GroupBy aggregation
regional_performance = df_deduped.groupby("region").agg(
    total_revenue=("transaction_amount", "sum"),
    average_ticket=("transaction_amount", "mean"),
    transaction_volume=("client_identifier", "count")
).reset_index()

# 2. Sorting by revenue descending
regional_performance = regional_performance.sort_values(
    by="total_revenue", 
    ascending=False
)

# 3. Multi-dimensional Pivot Table
pivot_summary = df_deduped.pivot_table(
    index="region",
    columns="order_state",
    values="transaction_amount",
    aggfunc="sum",
    fill_value=0.0,
    margins=True  # Generates "All" subtotal totals row/column
)

import pandas as pd

# Left DataFrame: Customer profiles
df_customers = pd.DataFrame({
    "cust_id": [101, 102, 103],
    "tier": ["Gold", "Silver", "Platinum"]
})

# Right DataFrame: Ledger records
df_orders = pd.DataFrame({
    "order_id": [5001, 5002, 5003],
    "account_num": [101, 101, 104],
    "val": [250.0, 400.0, 85.0]
})

# 1. Relational Merge with asymmetrical foreign key names
consolidated = pd.merge(
    df_customers,
    df_orders,
    left_on="cust_id",
    right_on="account_num",
    how="left",
    suffixes=("_customer", "_order")
)

# 2. Vertical Concatenation: Stacking monthly partitions (axis=0)
monthly_batches = [df_jan, df_feb, df_mar]
annual_dataset = pd.concat(monthly_batches, axis=0, ignore_index=True)

import pandas as pd

# 1. Standardize timestamp and set DatetimeIndex
df["timestamp"] = pd.to_datetime(df["timestamp"], format="%Y-%m-%d %H:%M:%S")
df = df.set_index("timestamp").sort_index()

# 2. Extract temporal components via .dt accessor
day_of_week = df.index.day_name()
is_weekend = df.index.weekday >= 5

# 3. Temporal Resampling: Aggregate raw telemetry into weekly buckets ('W')
weekly_sales = df["transaction_amount"].resample("W").sum()

# 4. Rolling Window: 7-day rolling moving average
df["rolling_7d_avg"] = df["transaction_amount"].rolling(window=7).mean()

# 5. Production Serialization & Exporting
df.to_csv("cleaned_transactions.csv", index=True)
df.to_excel("executive_summary.xlsx", sheet_name="Telemetry", index=True)
df.to_json("api_feed.json", orient="records", date_format="iso")
df.to_parquet("optimized_lake.parquet", engine="pyarrow", compression="snappy")