Working with Real-World Datasets

REAL-WORLD DATA SCIENCE WORKFLOW:

1. UNDERSTAND THE BUSINESS PROBLEM
   → What decision will this analysis support?
   → Who is the audience (technical/executive)?

2. DATA ACQUISITION
   → Kaggle API, database query, CSV export

3. DATA ASSESSMENT (30 minutes)
   → Shape, types, missing values, sample records

4. DATA CLEANING (often 60-70% of total time)
   → Duplicates, NAs, types, outliers, formats

5. EDA (analysis)
   → Univariate → Bivariate → Multivariate

6. INSIGHTS AND MODELING
   → Statistical tests, regression, ML

7. PROFESSIONAL REPORT
   → R Markdown → HTML/PDF report

library(dplyr); library(ggplot2); library(lubridate); library(scales)
library(readr); library(tidyr); library(stringr)

# Simulate realistic e-commerce transactions dataset
set.seed(42)
n <- 5000
products <- c("Laptop","Smartphone","Tablet","Headphones","Monitor","Keyboard","Mouse","Webcam")
categories <- c("Computers","Electronics","Computers","Audio","Computers","Accessories","Accessories","Electronics")
prod_cat <- setNames(categories, products)
prices_base <- c(Laptop=900, Smartphone=700, Tablet=450, Headphones=150,
                  Monitor=350, Keyboard=80, Mouse=40, Webcam=120)

transactions <- data.frame(
  order_id   = paste0("ORD", sprintf("%06d", 1:n)),
  date       = sample(seq(as.Date("2023-01-01"), as.Date("2024-12-31"), by="day"), n, TRUE),
  customer_id= paste0("CUST", sprintf("%05d", sample(1:2000, n, replace=TRUE))),
  product    = sample(names(prices_base), n, replace=TRUE,
                       prob=c(0.15,0.20,0.12,0.15,0.10,0.10,0.08,0.10)),
  quantity   = sample(1:5, n, replace=TRUE, prob=c(0.5,0.25,0.12,0.08,0.05)),
  discount_pct = sample(c(0,5,10,15,20), n, replace=TRUE, prob=c(0.4,0.2,0.2,0.1,0.1)),
  region     = sample(c("North","South","East","West"), n, replace=TRUE)
) %>%
  mutate(
    category     = prod_cat[product],
    base_price   = prices_base[product],
    unit_price   = base_price * (1 - discount_pct/100),
    revenue      = unit_price * quantity,
    cost         = base_price * 0.6 * quantity,
    profit       = revenue - cost,
    margin_pct   = round(profit/revenue*100, 1),
    year         = year(date),
    month        = month(date, label=TRUE),
    quarter      = paste0("Q", quarter(date))
  )

# ─── DATA QUALITY REPORT ─────────────────────────────
cat("=== E-COMMERCE DATA QUALITY ===\n")
cat("Orders:", nrow(transactions), "\n")
cat("Date Range:", as.character(min(transactions$date)), "to",
    as.character(max(transactions$date)), "\n")
cat("Unique Customers:", n_distinct(transactions$customer_id), "\n")
cat("Missing Values:", sum(is.na(transactions)), "\n\n")

# ─── EXECUTIVE KPI SUMMARY ───────────────────────────
cat("=== EXECUTIVE KPI SUMMARY ===\n")
kpis <- transactions %>%
  summarise(
    total_revenue = sum(revenue),
    total_profit  = sum(profit),
    avg_margin    = round(mean(margin_pct), 1),
    total_orders  = n(),
    unique_customers = n_distinct(customer_id),
    avg_order_value  = round(mean(revenue), 2),
    total_units      = sum(quantity)
  )
cat(sprintf("Total Revenue:     $%s\n", format(kpis$total_revenue, big.mark=",")))
cat(sprintf("Total Profit:      $%s\n", format(round(kpis$total_profit), big.mark=",")))
cat(sprintf("Avg Margin:        %.1f%%\n", kpis$avg_margin))
cat(sprintf("Total Orders:      %s\n", format(kpis$total_orders, big.mark=",")))
cat(sprintf("Unique Customers:  %s\n", format(kpis$unique_customers, big.mark=",")))
cat(sprintf("Avg Order Value:   $%.2f\n", kpis$avg_order_value))

# ─── TOP PRODUCTS ─────────────────────────────────────
cat("\n=== TOP PRODUCTS BY REVENUE ===\n")
transactions %>%
  group_by(product, category) %>%
  summarise(revenue=sum(revenue), profit=sum(profit),
            orders=n(), avg_margin=round(mean(margin_pct),1), .groups="drop") %>%
  arrange(desc(revenue)) %>%
  mutate(revenue=paste0("$",format(round(revenue), big.mark=",")),
         profit=paste0("$",format(round(profit), big.mark=","))) %>%
  print()

# ─── MONTHLY TREND ───────────────────────────────────
monthly <- transactions %>%
  group_by(year, month) %>%
  summarise(revenue=sum(revenue), orders=n(), .groups="drop") %>%
  mutate(date=as.Date(paste(year, as.integer(month), "01", sep="-")))

ggplot(monthly, aes(date, revenue, color=factor(year))) +
  geom_line(size=1.5) + geom_point(size=2) +
  scale_y_continuous(labels=dollar_format(scale=0.001, suffix="K")) +
  scale_x_date(date_breaks="2 months", date_labels="%b %Y") +
  scale_color_manual(values=c("2023"="#90CAF9","2024"="#1565C0"), name="Year") +
  labs(title="Monthly Revenue Trend", x="Month", y="Revenue") +
  theme_minimal() + theme(axis.text.x=element_text(angle=30, hjust=1))

# ─── REGIONAL ANALYSIS ────────────────────────────────
cat("\n=== REGIONAL PERFORMANCE ===\n")
transactions %>%
  group_by(region) %>%
  summarise(revenue=sum(revenue), profit=sum(profit),
            orders=n(), margin=round(mean(margin_pct),1), .groups="drop") %>%
  mutate(rev_share=round(revenue/sum(revenue)*100,1)) %>%
  arrange(desc(revenue)) %>% print()