diff --git a/README.md b/README.md index bd7496f84..17ba66226 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,7 @@ Full guide: `diff_diff.get_llm_guide("practitioner")`. - [Manipulation Testing](https://diff-diff.readthedocs.io/en/stable/api/regression_discontinuity.html) - Cattaneo, Jansson & Ma (2020) density-discontinuity test (`RDDensityTest`): rddensity 3.0 parity, robust bias-corrected inference, unrestricted/restricted models, mass-point adjustment - [Parallel Trends Testing](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html) - simple and Wasserstein-robust parallel trends tests, equivalence testing (TOST) - [Placebo Tests](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html) - placebo timing, group, permutation, leave-one-out +- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html) - Baker et al. (2025) implicit weights a TWFE regression places on each ATT(g,t), against the ATT^O / ATT^simple targets, with the pre-trend contribution. Ports Callaway's `twfeweights` (MIT) - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html) - Rambachan & Roth (2023) sensitivity analysis: robust CI under PT violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html) - Roth (2022) minimum detectable violation and power curves - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html) - analytical and simulation-based MDE, sample size, power curves for study design diff --git a/benchmarks/R/generate_twfeweights_golden.R b/benchmarks/R/generate_twfeweights_golden.R new file mode 100644 index 000000000..2666e8b1c --- /dev/null +++ b/benchmarks/R/generate_twfeweights_golden.R @@ -0,0 +1,510 @@ +#!/usr/bin/env Rscript +# Generate R `twfeweights` parity goldens for the diff-diff TWFE weight diagnostics. +# +# Requires: twfeweights (>= 0.9.0, MIT, Brantly Callaway), did, fixest, BMisc, +# DRDID, jsonlite +# Output: benchmarks/data/twfeweights_golden.json +# benchmarks/data/twfeweights_sim_panel.csv +# benchmarks/data/twfeweights_unbalanced_panel.csv +# +# The mpdta fixture reads the EXISTING benchmarks/data/mpdta_stata_panel.csv +# rather than writing a renamed copy of it; this script asserts the integer +# identifier columns match exactly and the float columns agree to CSV +# round-trip precision (1e-14 relative) before using it. +# +# Run from the repository root: +# Rscript benchmarks/R/generate_twfeweights_golden.R +# +# --------------------------------------------------------------------------- +# WHAT THIS PINS +# +# `diff_diff/twfe_weights.py` exposes two entry points, each folding several +# upstream R functions: +# +# attgt_weights(aggregation=) <- twfe_weights / attO_weights / +# att_simple_weights +# decompose_twfe_weights(method=) <- implicit_twfe_weights / +# implicit_aipw_weights +# +# plus covariate balance as a result-object method (<- twfe_cov_bal / +# aipw_cov_bal / mp_covariate_bal_summary_helper). +# +# RESERVED BLOCKS (pinned, but not yet read by any test): `decompose.aipw`, +# `balance.aipw` and `two_period.*` pin implicit_aipw_weights, aipw_cov_bal +# and the two_period_reg_weights / two_period_aipw_weights kernels. None of +# these has a Python surface yet - `method="aipw"` is a documented follow-up - +# so they are captured here so that follow-up needs no R re-run. Note the AIPW +# golden is COVARIATE-ADJUSTED: a time-invariant covariate is annihilated by +# double-demeaning but is NOT a no-op in a propensity score. +# +# --------------------------------------------------------------------------- +# NOTE (upstream bug — do NOT "simplify" the no-covariate calls below) +# +# twfeweights::implicit_twfe_weights() with xformula = ~1 (or NULL) builds +# model.matrix(BMisc::addCovToFormla("-1", ~1), data), an nT x 0 matrix, and +# fixest::demean() SEGFAULTS on a zero-column matrix. Reproduced in isolation +# on R 4.6.1 / fixest 0.14.2: +# +# fixest::demean(matrix(numeric(0), nrow = 10, ncol = 0), ids) +# *** caught segfault *** address ..., cause 'memory not mapped' +# +# This is a zero-column bug, NOT a conditioning problem with any particular +# fixture — every fixture hits it on the ~1 branch and no fixture hits it +# otherwise. +# +# The no-covariate decomposition is therefore generated by passing a +# TIME-INVARIANT covariate: double-demeaning annihilates such a regressor +# exactly, so the call is numerically identical to the ~1 branch. Verified on +# mpdta, where `lpop` is time-invariant within county: +# +# twfe_weights(att_gt(...)) aggregate = -0.03654894 +# implicit_twfe_weights(xformula = ~lpop) = -0.03654894 (exact) +# +# The Python parity test asserts BOTH covariates=None AND +# covariates=[] against this single golden, so the equivalence is +# proven by the test rather than assumed by the generator. +# --------------------------------------------------------------------------- + +suppressPackageStartupMessages({ + library(twfeweights) + library(did) + library(jsonlite) +}) + +stopifnot(packageVersion("twfeweights") == "0.9.0") + +# BMisc 1.4.9 emits .Deprecated warnings for makeBalancedPanel / addCovToFormla +# / getListElement on every internal call; they would otherwise flood the log. +quiet <- function(expr) suppressWarnings(suppressMessages(expr)) + +out_dir <- file.path("benchmarks", "data") +if (!dir.exists(out_dir)) { + stop("run this script from the repository root (", out_dir, " not found)") +} + +# --------------------------------------------------------------------------- +# Extraction helpers +# +# Field names below are read off the upstream S3 objects: +# mp_weights_obj $weights_df: group, time.period, weight, attgt, post +# decomposed_twfe $twfe_gt[[i]]: g, tp, weighted_outcome_diff (= ATT(g,t)), +# alpha_weight, ess, remainder, cov_bal_df +# decomposed_aipw $aipw_gt[[i]]: g, tp, est (= ATT(g,t)), att_weight, ess +# The scalar roll-ups mirror summary.decomposed_twfe / summary.decomposed_aipw +# exactly (twfeweights_mp.R:337 and :995). +# --------------------------------------------------------------------------- + +extract_mp_weights <- function(obj) { + df <- obj$weights_df + # mp_weights_obj stores `post` as a FACTOR (for ggplot colouring), so + # as.integer() would emit level codes 1/2 rather than the values 0/1. + # Round-trip through character. + post <- as.integer(as.character(df$post)) + stopifnot(all(post %in% c(0L, 1L))) + list( + group = as.numeric(df$group), + time = as.numeric(df$time.period), + post = post, + weight = as.numeric(df$weight), + att = as.numeric(df$attgt), + implied_att = sum(df$weight * df$attgt) + ) +} + +# implicit_* run in POSITIONAL time (BMisc::orig2t), so their $g / $tp are +# 1..T. attgt_weights' goldens carry RAW labels. Map back here so every block +# in the JSON shares one convention and the Python tests can assert labels. +to_orig <- function(pos, periods) { + out <- as.numeric(pos) + keep <- !is.na(out) & out >= 1 & out <= length(periods) + out[keep] <- as.numeric(periods[out[keep]]) + out +} + +extract_fwl <- function(obj, periods) { + cells <- obj$twfe_gt + g <- unlist(BMisc::getListElement(cells, "g")) + tp <- unlist(BMisc::getListElement(cells, "tp")) + att <- unlist(BMisc::getListElement(cells, "weighted_outcome_diff")) + wt <- unlist(BMisc::getListElement(cells, "alpha_weight")) + ess <- unlist(BMisc::getListElement(cells, "ess")) + rem <- unlist(BMisc::getListElement(cells, "remainder")) + post <- 1 * (tp >= g) + g <- to_orig(g, periods) + tp <- to_orig(tp, periods) + list( + cells = list( + group = as.numeric(g), time = as.numeric(tp), post = as.integer(post), + att = as.numeric(att), weight = as.numeric(wt), + ess = as.numeric(ess), remainder = as.numeric(rem) + ), + estimate = obj$est, + decomposition = obj$decomposition_est, + remainder = obj$decomposition_remainder, + pre_period_contribution = obj$pt_violations_bias, + post_only = sum(wt[post == 1] * att[post == 1]), + # summary.decomposed_twfe:351 + effective_sample_size = sum(post) * sum(wt[post == 1] * ess[post == 1]) + ) +} + +extract_aipw <- function(obj, periods) { + cells <- obj$aipw_gt + g <- unlist(BMisc::getListElement(cells, "g")) + tp <- unlist(BMisc::getListElement(cells, "tp")) + att <- unlist(BMisc::getListElement(cells, "est")) + wt <- unlist(BMisc::getListElement(cells, "att_weight")) + ess <- unlist(BMisc::getListElement(cells, "ess")) + post <- 1 * (tp >= g) + g <- to_orig(g, periods) + tp <- to_orig(tp, periods) + list( + cells = list( + group = as.numeric(g), time = as.numeric(tp), post = as.integer(post), + att = as.numeric(att), weight = as.numeric(wt), ess = as.numeric(ess) + ), + estimate = obj$est, + decomposition = obj$decomposition_est, + remainder = obj$decomposition_remainder, + pre_period_contribution = obj$pt_violations_bias, + post_only = sum(wt[post == 1] * att[post == 1]), + # summary.decomposed_aipw:1006 — note the inner sum is NOT post-filtered, + # unlike the twfe roll-up. Preserved verbatim. + effective_sample_size = sum(post) * sum(wt * ess) + ) +} + +# Per-cell balance tables, one row per (g, t) x covariate. +extract_balance_cells <- function(cells, periods) { + g <- unlist(BMisc::getListElement(cells, "g")) + tp <- unlist(BMisc::getListElement(cells, "tp")) + dfs <- BMisc::getListElement(cells, "cov_bal_df") + post_i <- 1 * (tp >= g) + g_o <- to_orig(g, periods) + tp_o <- to_orig(tp, periods) + rows <- do.call(rbind.data.frame, lapply(seq_along(dfs), function(i) { + d <- dfs[[i]] + cbind.data.frame( + group = g_o[i], time = tp_o[i], post = post_i[i], + covariate = rownames(d), d, row.names = NULL + ) + })) + as.list(lapply(rows, function(col) if (is.character(col)) col else as.numeric(col))) +} + +extract_balance_summary <- function(cells) { + s <- quiet(mp_covariate_bal_summary_helper(cells)) + out <- as.list(lapply(s, as.numeric)) + out$covariate <- rownames(s) + out +} + +extract_two_period <- function(obj) { + list( + estimate = as.numeric(obj$est), + ess = if (is.null(obj$ess)) NA_real_ else as.numeric(obj$ess), + weights = as.numeric(obj$weights), + dy = as.numeric(obj$dy), + treatment = as.numeric(obj$D) + ) +} + +# --------------------------------------------------------------------------- +# Per-fixture golden bundle +# +# `invariant_cov` must be TIME-INVARIANT within unit — it drives the +# no-covariate branch (see the segfault note at the top). +# `varying_cov` must be genuinely time-varying, so the covariate-adjusted +# weights differ from the unadjusted ones. +# --------------------------------------------------------------------------- + +build_fixture <- function(df, data_file, outcome, unit, time, first_treat, + invariant_cov, varying_cov, two_period_g, + data_file_out = NULL, columns_out = NULL, + derived_columns = NULL) { + stopifnot(all(tapply(df[[invariant_cov]], df[[unit]], + function(z) length(unique(z))) == 1)) + periods <- sort(unique(df[[time]])) + + # Slice the two-period sub-panel FIRST. Several upstream entry points + # (did::att_gt, and BMisc helpers reached from implicit_*) call + # data.table::setDT() on the frame they are handed, which converts it BY + # REFERENCE — after that, `df[cond, ]` silently takes data.table semantics + # and errors. Taking the subset up front sidesteps the whole problem. + tp_periods <- c(two_period_g - 1, two_period_g) + sub <- as.data.frame(df)[df[[time]] %in% tp_periods & + df[[first_treat]] %in% c(0, two_period_g), ] + + ag <- quiet(att_gt( + yname = outcome, tname = time, idname = unit, gname = first_treat, + xformla = ~1, data = df, control_group = "nevertreated", + base_period = "universal", bstrap = FALSE + )) + # did::att_gt calls data.table::setDT(data), which converts the caller's + # frame BY REFERENCE. Everything below assumes data.frame `[` semantics, so + # convert back explicitly rather than relying on what att_gt left behind. + df <- as.data.frame(df) + + inv_f <- as.formula(paste0("~", invariant_cov)) + var_f <- as.formula(paste0("~", varying_cov)) + bal_f <- as.formula(paste0("~", invariant_cov, "+", varying_cov)) + + common <- list(yname = outcome, tname = time, idname = unit, + gname = first_treat, data = df) + + fwl_nocov <- quiet(do.call(implicit_twfe_weights, + c(common, list(xformula = inv_f)))) + fwl_cov <- quiet(do.call(implicit_twfe_weights, + c(common, list(xformula = var_f)))) + fwl_gmin1 <- quiet(do.call(implicit_twfe_weights, + c(common, list(xformula = inv_f, + base_period = "gmin1")))) + aipw <- quiet(do.call(implicit_aipw_weights, + c(common, list(xformula = inv_f)))) + + # Balance is taken off the COVARIATE-ADJUSTED decomposition: on the + # no-covariate branch the implicit weights are constant within the treated + # and comparison groups, so weighted == unweighted and the table is + # degenerate (verified). fwl_cov gives a non-trivial reweighting. + bal_fwl <- quiet(twfe_cov_bal(fwl_cov, bal_f)) + bal_aipw <- quiet(aipw_cov_bal(aipw, bal_f)) + + # Two-period kernels: the (g = two_period_g) cohort against never-treated, + # over periods {g-1, g}. These are private in Python, so this is the only + # place they are pinned. (`sub` was sliced at the top of this function.) + sub_common <- list(yname = outcome, tname = time, idname = unit, + gname = first_treat, data = sub) + tp_reg <- quiet(do.call(two_period_reg_weights, + c(sub_common, list(xformula = var_f)))) + tp_aipw <- quiet(do.call(two_period_aipw_weights, + c(sub_common, list(xformula = var_f)))) + + out <- list( + data_file = if (is.null(data_file_out)) data_file else data_file_out, + columns = if (is.null(columns_out)) { + list(outcome = outcome, unit = unit, time = time, + first_treat = first_treat, + invariant_cov = invariant_cov, varying_cov = varying_cov) + } else { + columns_out + }, + two_period_group = two_period_g, + attgt_weights = list( + twfe = extract_mp_weights(quiet(twfe_weights(ag))), + overall = extract_mp_weights(quiet(attO_weights(ag))), + simple = extract_mp_weights(quiet(att_simple_weights(ag))) + ), + decompose = list( + fwl_nocov = extract_fwl(fwl_nocov, periods), + fwl_cov = extract_fwl(fwl_cov, periods), + fwl_gmin1 = extract_fwl(fwl_gmin1, periods), + aipw = extract_aipw(aipw, periods) + ), + balance = list( + fwl = list(cells = extract_balance_cells(bal_fwl$twfe_gt, periods), + summary = extract_balance_summary(bal_fwl$twfe_gt)), + aipw = list(cells = extract_balance_cells(bal_aipw$aipw_gt, periods), + summary = extract_balance_summary(bal_aipw$aipw_gt)) + ), + two_period = list(reg = extract_two_period(tp_reg), + aipw = extract_two_period(tp_aipw)) + ) + if (!is.null(derived_columns)) { + out$derived_columns <- derived_columns + } + out +} + +# --------------------------------------------------------------------------- +# Fixture 1 — mpdta (real data) +# +# did::mpdta: 500 US counties x 2003-2007, cohorts {2004, 2006, 2007} plus +# never-treated (first.treat == 0), time-invariant `lpop`. Also exercises +# non-1..T time labels, which the Python side handles by positional rescaling. +# --------------------------------------------------------------------------- + +data(mpdta, package = "did") +mpdta_df <- data.frame( + unit = as.numeric(mpdta$countyreal), + period = as.numeric(mpdta$year), + first_treat = as.numeric(mpdta$first.treat), + outcome = as.numeric(mpdta$lemp), + lpop = as.numeric(mpdta$lpop) +) +mpdta_df <- mpdta_df[order(mpdta_df$unit, mpdta_df$period), ] +# `lpop` is time-invariant; build a genuinely time-varying companion from it so +# the covariate-adjusted branch is non-degenerate on this fixture too. +mpdta_df$lpop_t <- mpdta_df$lpop * (mpdta_df$period - 2002) / 5 + +# The fixture READS benchmarks/data/mpdta_stata_panel.csv (already in the repo +# for the Stata parity suites) instead of writing a renamed copy. Assert the +# two sources agree bit-for-bit on every shared column, so they cannot drift. +stata_path <- file.path(out_dir, "mpdta_stata_panel.csv") +if (!file.exists(stata_path)) { + stop("expected ", stata_path, " (the mpdta fixture now reads it)") +} +stata_df <- read.csv(stata_path) +stata_df <- stata_df[order(stata_df$countyreal, stata_df$year), ] +# The identifiers must match exactly; the float columns are compared at CSV +# round-trip precision, NOT bit-for-bit. write.csv emits 15 significant digits, +# so a CSV column always sits within ~1e-15 relative of the in-memory double it +# came from. That gap is pre-existing and unchanged by this switch: the fixture +# previously read twfeweights_mpdta_panel.csv, itself a 15-digit round-trip of +# these same values, and the parity tolerances already absorb it. +rt_tol <- 1e-14 +stopifnot( + nrow(stata_df) == nrow(mpdta_df), + identical(as.numeric(stata_df$countyreal), mpdta_df$unit), + identical(as.numeric(stata_df$year), mpdta_df$period), + identical(as.numeric(stata_df$first.treat), mpdta_df$first_treat), + max(abs(as.numeric(stata_df$lemp) - mpdta_df$outcome)) <= + rt_tol * max(1, max(abs(mpdta_df$outcome))), + max(abs(as.numeric(stata_df$lpop) - mpdta_df$lpop)) <= + rt_tol * max(1, max(abs(mpdta_df$lpop))) +) + +# --------------------------------------------------------------------------- +# Fixture 2 — sim_staggered (simulated) +# +# 3 equal cohorts of 100, which keeps the AIPW propensity score bounded away +# from 0/1 (100 controls per cell). The equal cohorts are also exactly what +# makes the comparison-group normalizer VANISH at t = 3 (-1/3 + 1/3), so this +# fixture deliberately exercises the documented 0/0 cells - it is not a +# "no cell is degenerate" design. +# +# `0.3 * x1 * period` gives each unit a trend, so pre_period_contribution is non-zero, +# but x1 is iid and cohorts are assigned by unit INDEX, so E[x1 | g] does not +# vary by cohort: the differential pre-trend is zero in expectation and the +# observed value (~0.093) is sampling noise, not a designed pre-trend. +# `xtv`'s two structured terms (0.2 * period and 0.5 * x1) are absorbed by the +# two-way fixed effects, so the covariate branch regresses on the residual +# noise - adequate for parity, but not a "well-conditioned" design. +# --------------------------------------------------------------------------- + +make_sim <- function(seed, cohort_sizes, cohort_times, n_periods) { + set.seed(seed) + n <- sum(cohort_sizes) + g <- rep(cohort_times, times = cohort_sizes) + x1 <- rnorm(n) + unit_fe <- rnorm(n) + df <- do.call(rbind, lapply(seq_len(n_periods), function(t) { + data.frame(unit = seq_len(n), period = t, first_treat = g, + x1 = x1, unit_fe = unit_fe) + })) + df <- df[order(df$unit, df$period), ] + df$xtv <- 0.2 * df$period + 0.5 * df$x1 + rnorm(nrow(df), 0, 0.5) + treated <- (df$first_treat != 0) & (df$period >= df$first_treat) + df$outcome <- df$unit_fe + 0.5 * df$period + 0.3 * df$x1 * df$period + + 1.0 * treated * (df$period - df$first_treat + 1) + rnorm(nrow(df)) + df$unit_fe <- NULL + rownames(df) <- NULL + df +} + +sim_df <- make_sim(20260831, c(100, 100, 100), c(0, 3, 4), 5) + +# --------------------------------------------------------------------------- +# Fixture 3 — unbalanced_cohorts +# +# Fixture 2 has equal thirds, so p_g == 1/3 and several of the weight formulas +# coincide — a bug in the cohort-share computation would pass silently there. +# Unequal cohort masses (120 / 70 / 60) break that degeneracy. Do not drop this +# fixture. +# --------------------------------------------------------------------------- + +unb_df <- make_sim(20260901, c(120, 70, 60), c(0, 3, 5), 6) + +# --------------------------------------------------------------------------- +# Build + write +# --------------------------------------------------------------------------- + +write.csv(sim_df, file.path(out_dir, "twfeweights_sim_panel.csv"), + row.names = FALSE) +write.csv(unb_df, file.path(out_dir, "twfeweights_unbalanced_panel.csv"), + row.names = FALSE) + +cat("building mpdta ...\n") +fx_mpdta <- build_fixture( + mpdta_df, "twfeweights_mpdta_panel.csv", + "outcome", "unit", "period", "first_treat", + "lpop", "lpop_t", two_period_g = 2004, + # Emitted names point at the SHARED stata panel; the R calls above keep + # using mpdta_df's own names, so nothing inside build_fixture changes. + data_file_out = "mpdta_stata_panel.csv", + columns_out = list(outcome = "lemp", unit = "countyreal", time = "year", + first_treat = "first.treat", + invariant_cov = "lpop", varying_cov = "lpop_t"), + derived_columns = list(lpop_t = "lpop * (year - 2002) / 5") +) +cat("building sim_staggered ...\n") +fx_sim <- build_fixture(sim_df, "twfeweights_sim_panel.csv", + "outcome", "unit", "period", "first_treat", + "x1", "xtv", two_period_g = 3) +cat("building unbalanced_cohorts ...\n") +fx_unb <- build_fixture(unb_df, "twfeweights_unbalanced_panel.csv", + "outcome", "unit", "period", "first_treat", + "x1", "xtv", two_period_g = 3) + +payload <- list( + meta = list( + description = paste( + "R twfeweights parity goldens for diff_diff attgt_weights /", + "decompose_twfe_weights. Regenerate with:", + "Rscript benchmarks/R/generate_twfeweights_golden.R" + ), + upstream = paste( + "twfeweights (Brantly Callaway), MIT License,", + "Copyright (c) 2023 Brantly Callaway" + ), + r_version = paste(R.version$major, R.version$minor, sep = "."), + twfeweights_version = as.character(packageVersion("twfeweights")), + did_version = as.character(packageVersion("did")), + fixest_version = as.character(packageVersion("fixest")), + BMisc_version = as.character(packageVersion("BMisc")), + DRDID_version = as.character(packageVersion("DRDID")), + seeds = list(sim_staggered = 20260831L, unbalanced_cohorts = 20260901L), + mpdta_provenance = paste( + "fixtures.mpdta is data(mpdta, package = \"did\") version", + as.character(packageVersion("did")), + "- read from the shared benchmarks/data/mpdta_stata_panel.csv, whose", + "integer identifier columns this generator asserts are identical to", + "data(mpdta) and whose float columns it asserts agree to CSV round-trip", + "precision (1e-14 relative).", + "`lpop_t` is derived (see fixtures.mpdta.derived_columns)." + ), + reserved_blocks = paste( + "decompose.aipw, balance.aipw and two_period.* are PINNED BUT UNUSED:", + "they capture implicit_aipw_weights, aipw_cov_bal and the", + "two_period_reg_weights / two_period_aipw_weights kernels, none of which", + "has a Python surface yet (method=\"aipw\" is a documented follow-up).", + "They are kept so that follow-up needs no R re-run. NOTE the AIPW golden", + "is covariate-adjusted: a time-invariant covariate is annihilated by", + "double-demeaning but is NOT a no-op in a propensity score." + ), + label_convention = paste( + "Every cells block (attgt_weights.*, decompose.*, balance.*) carries", + "ORIGINAL period labels. implicit_* run in positional time internally;", + "the generator maps them back before emitting." + ), + no_covariate_note = paste( + "decompose.fwl_nocov is generated with xformula = ~,", + "which is numerically the ~1 branch (double-demeaning annihilates a", + "time-invariant regressor exactly). The ~1 branch itself cannot be", + "called: fixest::demean segfaults on the zero-column model matrix it", + "builds. See the comment block at the top of the generator." + ) + ), + fixtures = list( + mpdta = fx_mpdta, + sim_staggered = fx_sim, + unbalanced_cohorts = fx_unb + ) +) + +out_path <- file.path(out_dir, "twfeweights_golden.json") +write_json(payload, out_path, auto_unbox = TRUE, digits = NA, pretty = TRUE) +cat("wrote", out_path, "\n") +cat(" mpdta twfe implied_att =", fx_mpdta$attgt_weights$twfe$implied_att, "\n") +cat(" mpdta fwl_nocov estimate =", fx_mpdta$decompose$fwl_nocov$estimate, "\n") +cat(" sim fwl_nocov estimate =", fx_sim$decompose$fwl_nocov$estimate, "\n") +cat(" unb twfe implied_att =", fx_unb$attgt_weights$twfe$implied_att, "\n") diff --git a/benchmarks/R/requirements.R b/benchmarks/R/requirements.R index c85a33f4e..ba65b8cd7 100644 --- a/benchmarks/R/requirements.R +++ b/benchmarks/R/requirements.R @@ -20,6 +20,8 @@ required_packages <- c( "Synth", # Abadie-Diamond-Hainmueller (2010) synthetic control (SyntheticControl R-parity; ships data(basque)) "qte", # Callaway qte package (Athey-Imbens CiC + QDiD R-parity; ships data(lalonde)) "ptetools", # Callaway ptetools (badcontrols dependency; bad-control DMLDiD lane black-box parity) + "BMisc", # Callaway utility package (twfeweights dependency: weighted_ecdf, orig2t) + "DRDID", # Sant'Anna & Zhao (2020) doubly-robust DiD (twfeweights AIPW dependency) # Utilities "jsonlite", # JSON output for Python interop @@ -28,7 +30,9 @@ required_packages <- c( # synthdid must be installed from GitHub github_packages <- list( - synthdid = "synth-inference/synthdid" + synthdid = "synth-inference/synthdid", + # TWFE weight diagnostics parity goldens (not on CRAN) + twfeweights = "bcallaway11/twfeweights" ) install_if_missing <- function(pkg) { diff --git a/benchmarks/data/twfeweights_golden.json b/benchmarks/data/twfeweights_golden.json new file mode 100644 index 000000000..69502f40c --- /dev/null +++ b/benchmarks/data/twfeweights_golden.json @@ -0,0 +1,586 @@ +{ + "meta": { + "description": "R twfeweights parity goldens for diff_diff attgt_weights / decompose_twfe_weights. Regenerate with: Rscript benchmarks/R/generate_twfeweights_golden.R", + "upstream": "twfeweights (Brantly Callaway), MIT License, Copyright (c) 2023 Brantly Callaway", + "r_version": "4.6.1", + "twfeweights_version": "0.9.0", + "did_version": "2.5.1", + "fixest_version": "0.14.2", + "BMisc_version": "1.4.9", + "DRDID_version": "1.3.0", + "seeds": { + "sim_staggered": 20260831, + "unbalanced_cohorts": 20260901 + }, + "mpdta_provenance": "fixtures.mpdta is data(mpdta, package = \"did\") version 2.5.1 - read from the shared benchmarks/data/mpdta_stata_panel.csv, whose integer identifier columns this generator asserts are identical to data(mpdta) and whose float columns it asserts agree to CSV round-trip precision (1e-14 relative). `lpop_t` is derived (see fixtures.mpdta.derived_columns).", + "reserved_blocks": "decompose.aipw, balance.aipw and two_period.* are PINNED BUT UNUSED: they capture implicit_aipw_weights, aipw_cov_bal and the two_period_reg_weights / two_period_aipw_weights kernels, none of which has a Python surface yet (method=\"aipw\" is a documented follow-up). They are kept so that follow-up needs no R re-run. NOTE the AIPW golden is covariate-adjusted: a time-invariant covariate is annihilated by double-demeaning but is NOT a no-op in a propensity score.", + "label_convention": "Every cells block (attgt_weights.*, decompose.*, balance.*) carries ORIGINAL period labels. implicit_* run in positional time internally; the generator maps them back before emitting.", + "no_covariate_note": "decompose.fwl_nocov is generated with xformula = ~, which is numerically the ~1 branch (double-demeaning annihilates a time-invariant regressor exactly). The ~1 branch itself cannot be called: fixest::demean segfaults on the zero-column model matrix it builds. See the comment block at the top of the generator." + }, + "fixtures": { + "mpdta": { + "data_file": "mpdta_stata_panel.csv", + "columns": { + "outcome": "lemp", + "unit": "countyreal", + "time": "year", + "first_treat": "first.treat", + "invariant_cov": "lpop", + "varying_cov": "lpop_t" + }, + "two_period_group": 2004, + "attgt_weights": { + "twfe": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "weight": [-0.113075467453585, 0.0457198057404491, 0.0457198057404491, 0.0324868663076129, -0.0108510103349257, -0.0938215405788088, -0.107054480011645, -0.107054480011645, 0.197303126943588, 0.110627373658511, -0.0905761621829057, -0.133914038825444, -0.133914038825444, -0.220589792110522, 0.578994031944316], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286272, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], + "implied_att": -0.0365489366740672 + }, + "overall": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "weight": [0, 0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0, 0, 0, 0.104712041884817, 0.104712041884817, 0, 0, 0, 0, 0.68586387434555], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286272, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], + "implied_att": -0.031018282228749 + }, + "simple": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "weight": [0, 0.0687285223367698, 0.0687285223367698, 0.0687285223367698, 0.0687285223367698, 0, 0, 0, 0.13745704467354, 0.13745704467354, 0, 0, 0, 0, 0.450171821305842], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286272, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], + "implied_att": -0.039951275155177 + } + }, + "decompose": { + "fwl_nocov": { + "cells": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, 0, 0.00652011242423291, 0.00376929367371423, -0.000825313279148546, -0.0374551778725037, 0, 0.0305066555832921, 0.0277807626971762, -0.00330635669251204, -0.0293607674117092], + "weight": [-0.113075467453585, 0.0457198057404492, 0.0457198057404492, 0.0324868663076129, -0.0108510103349258, -0.0938215405788088, -0.107054480011645, -0.107054480011645, 0.197303126943588, 0.110627373658511, -0.0905761621829057, -0.133914038825444, -0.133914038825444, -0.220589792110521, 0.578994031944315], + "ess": [309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": -0.0365489366740672, + "decomposition": -0.0365489366740672, + "remainder": 0, + "pre_period_contribution": -0.00817768207864544, + "post_only": -0.0283712545954217, + "effective_sample_size": 2163 + }, + "fwl_cov": { + "cells": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "att": [0, -0.0133665432176392, -0.0704231581031491, -0.267917985756793, -0.0863164461874393, 0, 0.00207806356666779, 0.00376929367371422, -0.126820852395217, -0.0470847199292006, 0, 0.0267475453560011, 0.0277807626971762, -0.12743173909835, -0.0323931143733938], + "weight": [-0.1135230713667, 0.0464885411048002, 0.0461635736399947, 0.0324772245138304, -0.0116062678919253, -0.0912244608682691, -0.10633971008501, -0.108093577640393, 0.197464333015477, 0.108193415578195, -0.0876606039551204, -0.133316485481803, -0.135213842067535, -0.224628248535168, 0.580819180039627], + "ess": [288.006463639308, 296.614163218515, 309, 5.4990531247817, 304.859399183937, 288.006463639308, 296.614163218515, 309, 5.4990531247817, 304.859399183937, 288.006463639308, 296.614163218515, 309, 5.4990531247817, 304.859399183937], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": -0.0398490715024001, + "decomposition": -0.0398490715024001, + "remainder": 0, + "pre_period_contribution": 0.0206741188438872, + "post_only": -0.0605231903462873, + "effective_sample_size": 1650.8229562172 + }, + "fwl_gmin1": { + "cells": { + "group": [2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007, 2003, 2004, 2005, 2006, 2007], + "post": [0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1], + "att": [0, -0.0105032462209635, -0.0704231581031491, -0.137258738889404, -0.100811363085405, -0.00376929367371423, 0.00275081875051868, 0, -0.00459460695286273, -0.0412244715462179, 0.00330635669251199, 0.0338130122758041, 0.0310871193896881, 0, -0.0260544107191972], + "weight": [-0.113075467453585, 0.0457198057404492, 0.0457198057404492, 0.0324868663076129, -0.0108510103349258, -0.0938215405788088, -0.107054480011645, -0.107054480011645, 0.197303126943588, 0.110627373658511, -0.0905761621829057, -0.133914038825444, -0.133914038825444, -0.220589792110521, 0.578994031944315], + "ess": [309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309, 309], + "remainder": [-5.65463002249983, -5.65463002249983, -5.65463002249983, -5.65463002249983, -5.65463002249983, -5.60480843375399, -5.60480843375399, -5.60480843375399, -5.60480843375399, -5.60480843375399, -5.63889628205466, -5.63889628205466, -5.63889628205466, -5.63889628205467, -5.63889628205466] + }, + "estimate": -0.0365489366740662, + "decomposition": -0.0365489366740672, + "remainder": 9.57567358739198e-16, + "pre_period_contribution": -0.00893136238322589, + "post_only": -0.0276175742908413, + "effective_sample_size": 2163 + }, + "aipw": { + "cells": { + "group": [2004, 2004, 2004, 2004, 2006, 2006, 2007], + "time": [2004, 2005, 2006, 2007, 2006, 2007, 2007], + "post": [1, 1, 1, 1, 1, 1, 1], + "att": [-0.0145296683111164, -0.0764218817440481, -0.140448336820238, -0.106903898121729, 0.000960573746698802, -0.0412938655881805, -0.0287813610394872], + "weight": [0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0.0261780104712042, 0.104712041884817, 0.104712041884817, 0.68586387434555], + "ess": [293.350247392128, 293.350247392128, 293.350247392128, 293.350247392128, 253.063348228582, 253.063348228582, 295.099929127014] + }, + "estimate": -0.0328195971622414, + "decomposition": -0.0328195971622414, + "remainder": 0, + "pre_period_contribution": 0, + "post_only": -0.0328195971622414, + "effective_sample_size": 2002.7927075831 + } + }, + "balance": { + "fwl": { + "cells": { + "group": [2004, 2004, 2004, 2004, 2004, 2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2006, 2006, 2006, 2006, 2006, 2006, 2007, 2007, 2007, 2007, 2007, 2007, 2007, 2007, 2007, 2007], + "time": [2003, 2003, 2004, 2004, 2005, 2005, 2006, 2006, 2007, 2007, 2003, 2003, 2004, 2004, 2005, 2005, 2006, 2006, 2007, 2007, 2003, 2003, 2004, 2004, 2005, 2005, 2006, 2006, 2007, 2007], + "post": [0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1], + "covariate": ["mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t"], + "unweighted_covs_treated": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877, 3.45864088333128, 2.07518452999877], + "unweighted_covs_comparison": [3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216], + "unweighted_diff": [0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.275207332128624, 0.165124399277174, 0.275207332128624, 0.165124399277174, 0.275207332128624, 0.165124399277174, 0.275207332128624, 0.165124399277174, 0.275207332128624, 0.165124399277174], + "weighted_covs_treated": [3.4193925119911, 2.05163550719466, 3.54599912772186, 2.12759947663312, 3.47639737864719, 2.08583842718831, 3.37676805929306, 2.02606083557584, 4.03397261350373, 2.42038356810224, 3.63829301967841, 2.18297581180704, 3.70441978730257, 2.22265187238154, 3.75408740697126, 2.25245244418276, 3.72734009420077, 2.23640405652046, 3.6564541037253, 2.19387246223518, 2.98671356447979, 1.79202813868788, 3.30348590139512, 1.98209154083708, 3.45864088333128, 2.07518452999877, 3.55072510262075, 2.13043506157245, 3.38741487149293, 2.03244892289576], + "weighted_covs_comparison": [3.53045958121647, 2.11827574872988, 3.4460894159446, 2.06765364956676, 3.18343355120266, 1.9100601307216, 12.7324015067384, 7.63944090404307, 3.33323025444049, 1.99993815266429, 3.53045958121647, 2.11827574872988, 3.4460894159446, 2.06765364956676, 3.18343355120266, 1.9100601307216, 12.7324015067384, 7.63944090404307, 3.33323025444049, 1.99993815266429, 3.53045958121647, 2.11827574872988, 3.4460894159446, 2.06765364956676, 3.18343355120266, 1.9100601307216, 12.7324015067384, 7.63944090404307, 3.33323025444049, 1.99993815266429], + "weighted_diff": [-0.111067069225372, -0.066640241535223, 0.099909711777264, 0.059945827066358, 0.292963827444527, 0.175778296466716, -9.35563344744539, -5.61338006846723, 0.700742359063241, 0.420445415437945, 0.107833438461934, 0.0647000630771606, 0.258330371357971, 0.154998222814783, 0.570653855768603, 0.342392313461162, -9.00506141253767, -5.40303684752261, 0.32322384928481, 0.193934309570886, -0.54374601673668, -0.326247610042007, -0.142603514549474, -0.0855621087296847, 0.275207332128623, 0.165124399277174, -9.1816764041177, -5.50900584247062, 0.0541846170524449, 0.0325107702314671], + "sd": [1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.27801007041527, 0.766806042249162, 1.27801007041527, 0.766806042249162, 1.27801007041527, 0.766806042249162, 1.27801007041527, 0.766806042249162, 1.27801007041527, 0.766806042249162], + "unweighted_log_ratio_sd_diff": [-1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.40022970637138, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -1.14224374564749, -0.450837075241316, -0.450837075241315, -0.450837075241316, -0.450837075241315, -0.450837075241316, -0.450837075241315, -0.450837075241316, -0.450837075241315, -0.450837075241316, -0.450837075241315], + "weighted_log_ratio_sd_diff": [-2.16928559609177, -2.16928559609177, -1.69865356192303, -1.69865356192303, -1.40022970637138, -1.40022970637138, -4.92827389243409, -4.92827389243409, -0.738212307151284, -0.738212307151284, -2.24359876506552, -2.24359876506552, -1.8116712863662, -1.8116712863662, -1.14224374564749, -1.14224374564749, -4.44862281223905, -4.44862281223905, -1.84942446599253, -1.84942446599253, -1.54827828974358, -1.54827828974358, -1.33211844247205, -1.33211844247205, -0.450837075241316, -0.450837075241316, -3.45912678775289, -3.45912678775289, -0.922554990848926, -0.922554990848926], + "unweighted_frac_treated_extreme": [0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.05, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.075, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764, 0.0610687022900764], + "weighted_frac_treated_extreme": [0, 0, 0, 0, 0.05, 0.05, 0, 0, 0.2, 0.2, 0, 0, 0, 0, 0.075, 0.075, 0, 0, 0, 0, 0.0229007633587786, 0.0229007633587786, 0.0229007633587786, 0.0229007633587786, 0.0610687022900764, 0.0610687022900764, 0, 0, 0.0305343511450382, 0.0305343511450382] + }, + "summary": { + "unweighted_treat": [3.55096217446187, 2.13057730467712], + "unweighted_untreat": [3.18343355120266, 1.91006013072159], + "unweighted_diff": [0.367528623259216, 0.22051717395553], + "weighted_treat": [3.48727645229424, 2.09236587137654], + "weighted_untreat": [5.49282183830626, 3.29569310298376], + "weighted_diff": [-2.00554538601202, -1.20332723160721], + "sd": [1.27667557327224, 0.766005343963346], + "unweighted_log_ratio_sd_diff": [-0.769948848899112, -0.769948848899111], + "weighted_log_ratio_sd_diff": [-1.90947382058374, -1.90947382058374], + "unweighted_frac_treated_extreme": [0.0640703583030666, 0.0640703583030666], + "weighted_frac_treated_extreme": [0.0177218618987178, 0.0177218618987178], + "covariate": ["mean_lpop", "mean_lpop_t"] + } + }, + "aipw": { + "cells": { + "group": [2004, 2004, 2004, 2004, 2004, 2004, 2004, 2004, 2006, 2006, 2006, 2006, 2007, 2007], + "time": [2004, 2004, 2005, 2005, 2006, 2006, 2007, 2007, 2006, 2006, 2007, 2007, 2007, 2007], + "post": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "covariate": ["mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t", "mean_lpop", "mean_lpop_t"], + "unweighted_covs_treated": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877], + "unweighted_covs_comparison": [3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216, 3.18343355120266, 1.9100601307216], + "unweighted_diff": [0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.292963827444527, 0.175778296466716, 0.570653855768603, 0.342392313461162, 0.570653855768603, 0.342392313461162, 0.275207332128624, 0.165124399277174], + "weighted_covs_treated": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877], + "weighted_covs_comparison": [3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.47639737864719, 2.08583842718831, 3.75408740697126, 2.25245244418276, 3.75408740697126, 2.25245244418276, 3.45864088333128, 2.07518452999877], + "weighted_diff": [8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 4.44089209850063e-16, 8.88178419700125e-16, 8.88178419700125e-16], + "sd": [1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.2847982850936, 0.770878971056158, 1.271122902882, 0.762673741729201, 1.271122902882, 0.762673741729201, 1.27801007041527, 0.766806042249162] + }, + "summary": { + "unweighted_treat": [3.52237381972876, 2.11342429183725], + "unweighted_untreat": [3.18343355120266, 1.9100601307216], + "unweighted_diff": [0.338940268526096, 0.203364161115658], + "weighted_treat": [3.52237381972876, 2.11342429183725], + "weighted_untreat": [3.52237381972875, 2.11342429183725], + "weighted_diff": [8.88178419700125e-16, 7.4867395587288e-16], + "sd": [1.27727853948457, 0.766367123690741], + "unweighted_log_ratio_sd_diff": [0, 0], + "weighted_log_ratio_sd_diff": [0, 0], + "unweighted_frac_treated_extreme": [0, 0], + "weighted_frac_treated_extreme": [0, 0], + "covariate": ["mean_lpop", "mean_lpop_t"] + } + } + }, + "two_period": { + "reg": { + "estimate": -0.0149216931463987, + "ess": 295.437073557126, + "weights": [0.914205378751222, 1.10807424931814, 0.933944555836912, 1.30885089779359, 0.77667566709387, 1.09886549432803, 0.857802290117748, 1.29533608776587, 1.21797362329684, 0.890296869666088, 0.876826211579155, 1.22341271500028, 0.861220240964523, 1.2217569875752, 1.23432024149811, 1.10053686950126, 1.53243455097736, 0.991986984922291, 0.852012353181866, 0.892799434382799, 1.15625878851813, 1.01070586338101, 0.797104804854826, 0.82150263374418, 0.925758568885049, 1.06901911216371, 1.15731335732235, 1.20326032975876, 0.991885473966669, 1.07626137455175, 0.958503061349015, 0.921310469219996, 0.781305672392859, 0.880672266280691, 0.868599070143875, 1.15374257218318, 0.978235389547607, 1.20698785760044, 0.864242279469781, 0.984348590136261, 0.677018937975777, 0.78566066241169, 1.07051467897138, 0.851002176458188, 0.800796993125892, 0.873966920756733, 0.922755161442552, 0.970155022871146, 0.906505733152204, 1.16334110821022, 1.00654509305044, 1.00714845007322, 1.00861119030535, 1.01139099432848, 1.01655775675883, 0.999377477471499, 1.00885020589396, 0.99839371957829, 0.972753520276552, 0.967719834724691, 0.983415466966356, 1.00756804487529, 1.00057938242599, 1.01802260153828, 0.983491689143243, 1.0162255160028, 1.00802635808135, 0.993215960705129, 0.993102181707199, 0.999004556093054, 1.18092367771598, 1.08808685196963, 1.23100981435726, 1.0159731214296, 1.05650610397299, 1.10794776362609, 1.08543904432851, 1.26567040981729, 1.18545393782311, 1.24368399209914, 1.26059743946738, 1.08112581656952, 1.25304541183838, 1.10727033756971, 1.10436293348918, 1.1694637092217, 1.10854681715011, 0.757196653021297, 0.98296939076698, 1.06678038285255, 1.09802731714199, 1.30342090571583, 1.02010825188693, 1.02240090966505, 1.0076789351107, 1.27816316617544, 0.870787994093949, 0.864507234249911, 0.874319684458889, 0.958503061349015, 0.971488264668903, 1.07593314326871, 1.34547694644207, 0.987118006851055, 0.867133874241953, 0.880346868666857, 0.92188543431963, 1.45715035175091, 0.958529642604906, 0.752567595243238, 0.97420384402187, 0.787153467781112, 1.08523202059903, 0.864818407544575, 0.989121047467235, 0.832273511291086, 0.833084041058782, 1.23693372233058, 0.675719067675065, 0.656205952440136, 0.89244421716068, 0.829434045397887, 1.17439425571453, 0.941590453075537, 0.766703290704483, 0.772482352776408, 0.799168669044801, 0.74157547066872, 0.721387168342452, 0.858610943520528, 0.994797353734057, 1.47335777302894, 1.18514252521156, 1.0803767751372, 1.50009361600496, 1.30150183654471, 0.846743269804263, 1.17080805501646, 0.861696060941864, 0.990574796380501, 1.12968407360812, 1.09171654630869, 0.804140956149301, 0.831231009547144, 0.772076141498604, 0.850308351294569, 0.880658131758654, 1.49267787652659, 0.784758519879128, 0.681653607455449, 0.596392417171735, 1.13283686776408, 0.8429053802999, 0.691259374141176, 0.833271972854951, 0.842390476649212, 1.38993252780947, 0.75168132143843, 0.806891915601875, 0.925099414970179, 0.65723510736642, 0.845159246065923, 0.90737500976245, 1.27095562512986, 0.967288076540284, 0.773426347208265, 0.958219263901373, 1.00982894404555, 1.32997931908278, 1.13888195173276, 0.74351757596311, 1.25854765822588, 0.628166041980897, 0.761122527803733, 0.606941516111651, 1.16800760300955, 0.64339210703374, 0.767652647794539, 0.784683120487471, 0.688103599457281, 0.764929768007258, 0.825623696972631, 0.888876693782396, 0.966707035387086, 0.770906152874736, 1.06914691812822, 1.03713164810658, 1.21511287945244, 0.660496935454807, 1.01335391953086, 0.91107858003953, 0.86150589590303, 0.856908216912444, 1.05679134782963, 1.17622845309429, 1.17317400146166, 1.0970151373967, 1.52515362690788, 1.1447241641711, 0.955589011684891, 1.32121999096743, 0.938232159752092, 1.2686858175866, 0.922799077792815, 1.42558003259381, 1.05749743636383, 1.09540926270813, 1.00318548722524, 1.34970798424194, 1.13005176807204, 1.17650664269292, 1.06420125597473, 1.16827467058984, 0.961741161438828, 1.03563770711794, 1.06417305269481, 0.565642036368166, 1.01178430075611, 0.80013382343569, 0.804163323876345, 0.93645145535438, 0.498752941366539, 0.873185246178276, 0.591658673083957, 0.671509856439639, 0.85343303820078, 0.804185688602419, 0.543035144130157, 0.836526939309612, 0.937016901846683, 0.888580417998956, 1.24643165992158, 1.0836459985709, 1.0550178960445, 1.02953012986436, 1.42446146773525, 1.13292714422746, 0.986819202405292, 1.01935952633555, 1.08394687794462, 1.28673644369391, 1.12770172727897, 1.60279560849242, 0.94803517304973, 0.94936028579014, 0.996230905422344, 0.947827464556498, 1.0489182301759, 0.824459438478706, 0.833102843775575, 1.04806606792118, 1.43863822710714, 1.10992438468713, 0.869145215288365, 1.75368577552688, 0.956263947765851, 1.48129133194083, 0.714177591294102, 0.526137284043039, 1.18516755063542, 1.08877777658533, 1.04685002787374, 1.39110059759066, 1.08142279773759, 0.672301055720514, 1.11637172339878, 1.17790730294754, 0.655607208827931, 0.782634337497015, 0.728931237851374, 1.1121882817121, 0.969716513585689, 0.921553965756354, 0.8357145095486, 1.01889756947789, 0.85343303820078, 0.669665695616983, 0.65506102391745, 1.09662092099898, 0.876391731353484, 1.17058226607269, 0.935233286711417, 0.668458481030845, 0.571884667288901, 0.673529835319247, 1.10335516966084, 1.03836004925978, 1.04917045987158, 0.950619215471048, 0.88606795324462, 0.87790748338468, 1.09531959221919, 1.38223356446573, 0.820856369845608, 0.914101320529017, 1.22008833913762, 1.19820046386571, 1.16675729242193, 0.739214676891295, 1.15287231341345, 1.0390337443305, 0.954983748994095, 1.3962143210842, 1.05844399017882, 1.00276474444757, 1.61570706617249, 1.13776110303408, 0.950377493029127, 0.92344550363475, 1.14599221194468, 1.01003722393432, 0.89876316093955, 1.20719528706948, 1.05242527002734, 1.10239361278195, 0.776120851588995, 0.94518023966827, 0.962590677452325, 1.30057892353915, 1.08619456613142, 1.16587595058709, 1.33511731721303, 1.37900024035138, 0.877418175728803, 1.22829047749297, 0.998079075301512, 1.16175961199361, 0.883530934105611, 0.996195569866188], + "dy": [0.0402064204780404, -0.00617921636595753, -0.00769234562315635, -0.0109482153338982, 0.271933715483642, 0.0670107102829602, -0.159064694629687, 0.0776028399652979, -0.0348005291494164, -0.361790044605503, -0.023530497410194, 0.0407981988772361, -0.039441732051297, 0.0183057960884394, 0.0578857923998042, -0.0689928714869517, -0.0489120485013519, -0.141078598259906, -0.231801614057324, -0.0979804083602032, -0.107245530353598, 0.0255333020051651, -0.100083458556983, -0.42744401482694, -0.00557104504945549, -0.0268472500361883, 0.0447260357486368, 0.0994408366375259, 0.115069329784787, -0.0392207131532816, 0.0330062964681703, -0.188348264660815, 0.37469344944141, 0.185403223331362, 0.105360515657826, -0.123805501162851, -0.127561225638019, 0.00391083799199876, -0.182321556793955, 0.0138891121606672, 0.352821374622742, 0.0202027073175195, 0.00183992692200707, -0.0615578929994332, -0.145851877012563, -0.101020402315209, -0.0900261516577192, 0.00645163528148807, 0.0823995317670665, -0.0217808084613926, -0.118481603620523, -0.00881062968215485, -0.0836500292869244, -0.332133835022615, 0.0377403279828474, -0.0805032188490706, -0.127833371509885, -0.103666774395267, -0.0391226547103702, -0.0522125710903065, -0.114421687953678, 0.00524935588614373, 0.024916351264535, -0.287682072451781, -0.052592450119171, 0.0108109161042158, -0.0619667237491992, -0.049576446606288, 0.0494724535390656, -0.0782007473074744, -0.0358461317731358, -0.0870113769896301, -0.0922165496035534, 0.264301623807717, -0.0495596909484606, -0.0583355770129632, -0.0856554445784941, -0.11963720201451, -0.0900336686743657, 0.0372967877532266, -0.0117011085851217, -0.081526607634208, -0.0620637099045016, -0.0557419830082351, -0.0589674760881156, -0.0501712931478435, -0.033953878930344, -0.0277795641070755, 0.0201011793210872, 0.0344321964214842, -0.0153377239837091, -0.0636388359666737, -0.0337381396318506, -0.136464423894526, -0.210379509589112, -0.0663859260598123, -0.24231346742193, -0.0682679711290026, -0.0795120629277335, -0.0880333494853227, -0.191268342783761, -0.0861116918187319, -0.0228979640932554, -0.0592556962090933, -0.0804723532225733, 0.0714589639821446, -0.0876299974856956, -0.0657078415693917, -0.249811798396371, -0.111791405988116, -0.168898536461814, -0.316669609325034, -0.0500409460710847, 0.0302398851897179, -0.234400705838844, 0.094778406327289, -0.0897699994287091, -0.155955083059526, 0.143100843640673, -0.0730251350148903, -0.0628009012390303, -0.0145987994211527, -0.00685403642779203, -0.189888919858863, -0.0266682470821618, 0.037740327982847, -0.0111733005981254, -0.117783035656383, -0.0384662808277962, 0, 0.0571584138399484, -0.0694119283885524, -0.0735133828982697, -0.104572803919577, -0.174540558863677, -0.159272329947313, -0.241162056816889, -0.151592882072704, 0.336472236621213, -0.364897923119164, 0.03964483134724, -0.229033345370273, -0.182321556793954, -0.251314428280907, 0.127833371509885, 0.0752830604205483, -0.0685626714372969, -0.0333465799413801, -0.197610249309045, -0.102654154060084, 0.0645385211375711, -0.071686734959413, 0.0268112574506567, -0.11247798342669, -0.116682320080198, 0.143100843640673, -0.0513428079171341, -0.0591888713903304, 0.0285733724440558, -0.2383103438577, -0.0444517625708336, 0.0909717782057271, -0.220542769614153, -0.03650636928728, -0.00735297430525872, -0.136859182717197, -0.0115943327809198, -0.306619273471531, -0.0180556878262053, -0.0133740595252565, -0.847297860387203, -0.0694513863237711, -0.0165293019512109, -0.391671785975829, 0, -0.0750083720980301, -0.246860077931526, -0.351397886837889, -0.23211222129697, 0.131028262406404, 0.0389154162496732, -0.0231224174208542, -0.0749976519277089, -0.0967787628858519, -0.264151575041587, 0.0642791514045653, -0.181470854913642, -0.035350988878931, -0.22314355131421, -0.0585533087515744, -0.170625517030763, -0.117783035656384, -0.223143551314211, -0.271496748030764, -0.231840227823137, -0.0830516576038791, -0.0348753293140129, -0.0905435929019962, -0.039441732051297, -0.091434205959632, -0.0468571586878825, -0.0959639882617438, -0.0117859422190874, -0.0923733201310153, -0.0130171058024899, 0.00475060275859818, -0.0717936920187423, -0.130053128248198, -0.0133312519554591, 0.124674861694305, -0.163954679923878, -0.0845573880280623, -0.132324877296182, 0.0560894666510432, -0.208466836898277, -0.0640702467202106, -0.0487901641694322, -0.0536210914390978, -0.0444517625708345, 0.0861776962410516, -0.00652885888246324, 0, -0.146379535102372, 0.310154928303839, -0.0176995770994006, 0.0115608224010764, 0.100470530363635, 0.162518929497775, 0.264692554227082, -0.113077144618971, -0.120144311842064, 0.059033531584193, 0.00392927813988919, -0.124850202879476, 0.00626961301359508, 0.0298084662978262, -0.118055998671591, -0.131576357788719, -0.00881062968215485, 0.0217399866364056, 0.0184337016888385, 0.079111001652536, -0.038374012881615, -0.305864391534971, -0.167054084663166, 0.126675144754063, -0.0807613566444587, 0.0213911899813182, 0.030459207484709, 0.0540672212702757, 0.145310091817136, -0.025075940768212, -0.274987152970533, -0.232931557680373, -0.103570370042048, -0.141746288743414, 0.00681006216133007, 0.0512932943875506, -0.606135803570316, -0.0969922659873097, -0.0986425361174703, -0.0369435151916848, -0.094288947978324, 0.141650517063027, -1.02961941718116, -0.0822893947565149, -0.0191944472561474, 0.328504066972036, -0.0706175672139531, -1.46633706879343, -0.334639055539552, -0.154681606306414, 0.133531392624523, -0.0668629452291265, 0.0144930073025664, 0.120627987788614, 0.176930708159078, -0.382992252256106, 0.0499410291375364, -0.113944259349218, 0.0173917427118688, -0.168059109873391, -0.0281708769666964, -0.367724780125317, -0.470003629245735, -0.188221278921143, 0, -0.0435712482936399, 0.0165293019512109, 0.255933374137201, 0.0327898228229908, -0.052471051737454, -0.0284494118748899, -0.0729489996461679, 0.112618878108149, -0.0134602190556974, 0.0395764757017059, -0.0756595628451411, -0.492476485097794, 0.0179109265665307, -0.0551320706947163, -0.101096116871369, 0.0107473405956728, 0.092721740050222, -0.034635496662756, -0.00818102901645723, 0.0590490288528001, 0.202236866494896, -0.402992918962775, -0.195516044873487, -0.0569081322718796, -0.0377403279828474, -0.0691272710570754, -0.112417707967825, -0.188742124596877, 0.0428199971829288, 0.0607562094210721, 0.101747438973162, 0.000974184197843009, -0.0126184639592113, -0.0388979479328331, 0.0638081142684905, -0.0234736356127776, 0.0601685214664531, -0.07417986546408, -0.0202436062766465, 0.0964137296680203, -0.0302330106601936, 0.0640072185482552], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "aipw": { + "estimate": -0.0145296683111164, + "ess": 293.350247392128, + "weights": [0.894002251049714, 1.08912887056764, 0.912139448798106, 1.33663687068034, 0.777353982575372, 1.07895492287744, 0.844159798828147, 1.3183278848748, 1.21827258329935, 0.872521288429006, 0.86064891071784, 1.22505040897046, 0.847098610659438, 1.22298314584019, 1.23875720803122, 1.08079433857419, 1.67945989357214, 0.967653605894451, 0.839205021306279, 0.874745094998973, 1.14396143122207, 0.986274905763588, 0.793655426945849, 0.813577909783472, 0.904573126031373, 1.04663392636343, 1.14519210123462, 1.20012641825204, 0.967553597396161, 1.05438584858977, 0.935224694727841, 0.900488358508805, 0.781018532914758, 0.864021879677238, 0.853478380599116, 1.14103041278715, 0.954200123207928, 1.2046977512144, 0.849705627797744, 0.960157332889347, 0.702565586174883, 0.784481381193752, 1.04823001919817, 0.838343574206719, 0.796638337684592, 0.858149958258436, 0.901812997158555, 0.946383153027563, 0.887026666261882, 1.15225205545184, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1.17309689736406, 1.06716813845226, 1.23458096954703, 0.99157960228583, 1.03337549477273, 1.08898847481201, 1.06429258825978, 1.27901832490392, 1.17852895390884, 1.25064709755962, 1.27241523475185, 1.05962508654199, 1.26264882346518, 1.08823686024391, 1.0850169829417, 1.15946797717608, 1.08965356969476, 0.762126536933417, 0.95881005552149, 1.04424929648179, 1.07803366329225, 1.32925013731868, 0.995764287470606, 0.998092093051924, 0.983239427591536, 1.29542652220053, 0.855380262678252, 0.849934582420623, 0.858457868237986, 0.935224694727841, 0.9476684674978, 1.05403326965604, 1.3875512537957, 0.962868424186321, 0.852207710912963, 0.863735990959885, 0.901015308497714, 1.55514398580636, 0.935249998401912, 0.758552494022138, 0.95029188025848, 0.785671952662053, 1.06406808753133, 0.850203555377142, 0.964834093590168, 0.822533389701864, 0.823211329274651, 1.24206426011762, 0.701639876047414, 0.687891345473986, 0.874429096523289, 0.820162869388286, 1.16531204434823, 0.919264300892949, 0.769520096761839, 0.774050093411802, 0.795321421756718, 0.750133494599614, 0.734916916052496, 0.844854160183868, 0.970426525099808, 1.58110223940849, 1.17815474202094, 1.05881662670873, 1.6248775532266, 1.32664934577575, 0.834721535320327, 1.16105846772014, 0.847508548356641, 0.966263255565178, 1.11338493606689, 1.07112275936809, 0.799349711866075, 0.821662255653284, 0.773730798373221, 0.837752419995297, 0.864009459343324, 1.61261501076916, 0.78376277257012, 0.705876292742996, 0.647428686090855, 1.11696902174642, 0.831471116458873, 0.712788525836175, 0.823368599043228, 0.831036004982153, 1.45197650533514, 0.757870151528899, 0.801587265560697, 0.903966634067484, 0.688609587838628, 0.833378425203411, 0.88781143556202, 1.2859343470801, 0.943625225641593, 0.774792618745101, 0.934954580596145, 0.985394537886997, 1.36577400696854, 1.12387359379345, 0.751614058902251, 1.26975693234241, 0.668612420905503, 0.765171028921774, 0.654385132001179, 1.1577477383766, 0.679012060093675, 0.77026241566466, 0.783702742793706, 0.710510112907514, 0.768135289274691, 0.816992668722927, 0.871261846989218, 0.94306726889844, 0.772811895241902, 1.04677022713451, 1.01318017562438, 1.21472287797626, 0.690891037043235, 0.988938195082353, 0.891162830885818, 0.847344689605216, 0.84339276238123, 1.03367583677142, 1.16749363935029, 1.16386294782544, 1.07692221020635, 1.66701335998637, 1.13058730375163, 0.932454891509071, 1.35361774608789, 0.916127968293349, 1.28295957022046, 0.901853294887475, 1.50580256277375, 1.0344196777086, 1.07516120251468, 0.978750662101694, 1.39355713356033, 1.11380233235004, 1.16782487713884, 1.04150887454439, 1.15806305850471, 0.938312253912984, 1.01163956811404, 1.04147894758405, 0.627576820467188, 0.987358672362671, 0.796101729352255, 0.799367879632361, 0.914469343531904, 0.586506346702584, 0.85746807099136, 0.644331572783776, 0.698650747799085, 0.840418052265205, 0.799386045379082, 0.613378159731018, 0.826097329094868, 0.914995695834427, 0.87099933523313, 1.25415777108945, 1.06234975602779, 1.03180994165805, 1.00536574424389, 1.50408349546532, 1.11707181872119, 0.962575541260206, 0.99500527053113, 1.06267552030451, 1.306809166267, 1.11113734646778, 1.80465227361454, 0.925313475141831, 0.926562233085535, 0.971844061197384, 0.925117889238856, 1.02541821473023, 0.816026483828058, 0.823227062864221, 1.02452843411696, 1.52601767939589, 1.0911845575514, 0.853952506965403, 2.10561116586107, 0.93309568004779, 1.59396704068207, 0.729558976902732, 0.602979072986013, 1.17818480963529, 1.06791977822604, 1.02326006063466, 1.45370921722291, 1.05994579825869, 0.69921162182759, 1.09837880259324, 1.16949406859886, 0.687473835164664, 0.782073370173473, 0.740566165391499, 1.09370529557565, 0.945960791989393, 0.900711482725538, 0.82541539067686, 0.994537255170615, 0.840418052265205, 0.697345210807313, 0.687093199004881, 1.07648964211119, 0.860268711765294, 1.16079118422479, 0.913336436309487, 0.696491928549023, 0.631556234278697, 0.700083598765149, 1.08390315180698, 1.01444872130449, 1.02568172844303, 0.927750198039274, 0.868776406041439, 0.8615958327451, 1.07506295514825, 1.44060768659023, 0.813043719924969, 0.893907609746538, 1.22090330227284, 1.19394906972845, 1.15627267688281, 0.748337722818574, 1.14001845186309, 1.01514511301218, 0.931880629247007, 1.46131939576755, 1.03541768771894, 0.97833141796237, 1.82862252736225, 1.12259014409163, 0.927521982019721, 0.902446667017561, 1.13204983708822, 0.985603565025875, 0.880067648926837, 1.20495265096482, 1.02908828203376, 1.0828414651844, 0.776916027928803, 0.922628819453102, 0.939123978418286, 1.32540040382158, 1.06511229655857, 1.15523404587142, 1.37295557236873, 1.43585985120983, 0.861167191951046, 1.23116101723212, 0.973674667516388, 1.15039551027658, 0.866537571207786, 0.971809095301766], + "dy": [0.0402064204780404, -0.00617921636595753, -0.00769234562315635, -0.0109482153338982, 0.271933715483642, 0.0670107102829602, -0.159064694629687, 0.0776028399652979, -0.0348005291494164, -0.361790044605503, -0.023530497410194, 0.0407981988772361, -0.039441732051297, 0.0183057960884394, 0.0578857923998042, -0.0689928714869517, -0.0489120485013519, -0.141078598259906, -0.231801614057324, -0.0979804083602032, -0.107245530353598, 0.0255333020051651, -0.100083458556983, -0.42744401482694, -0.00557104504945549, -0.0268472500361883, 0.0447260357486368, 0.0994408366375259, 0.115069329784787, -0.0392207131532816, 0.0330062964681703, -0.188348264660815, 0.37469344944141, 0.185403223331362, 0.105360515657826, -0.123805501162851, -0.127561225638019, 0.00391083799199876, -0.182321556793955, 0.0138891121606672, 0.352821374622742, 0.0202027073175195, 0.00183992692200707, -0.0615578929994332, -0.145851877012563, -0.101020402315209, -0.0900261516577192, 0.00645163528148807, 0.0823995317670665, -0.0217808084613926, -0.118481603620523, -0.00881062968215485, -0.0836500292869244, -0.332133835022615, 0.0377403279828474, -0.0805032188490706, -0.127833371509885, -0.103666774395267, -0.0391226547103702, -0.0522125710903065, -0.114421687953678, 0.00524935588614373, 0.024916351264535, -0.287682072451781, -0.052592450119171, 0.0108109161042158, -0.0619667237491992, -0.049576446606288, 0.0494724535390656, -0.0782007473074744, -0.0358461317731358, -0.0870113769896301, -0.0922165496035534, 0.264301623807717, -0.0495596909484606, -0.0583355770129632, -0.0856554445784941, -0.11963720201451, -0.0900336686743657, 0.0372967877532266, -0.0117011085851217, -0.081526607634208, -0.0620637099045016, -0.0557419830082351, -0.0589674760881156, -0.0501712931478435, -0.033953878930344, -0.0277795641070755, 0.0201011793210872, 0.0344321964214842, -0.0153377239837091, -0.0636388359666737, -0.0337381396318506, -0.136464423894526, -0.210379509589112, -0.0663859260598123, -0.24231346742193, -0.0682679711290026, -0.0795120629277335, -0.0880333494853227, -0.191268342783761, -0.0861116918187319, -0.0228979640932554, -0.0592556962090933, -0.0804723532225733, 0.0714589639821446, -0.0876299974856956, -0.0657078415693917, -0.249811798396371, -0.111791405988116, -0.168898536461814, -0.316669609325034, -0.0500409460710847, 0.0302398851897179, -0.234400705838844, 0.094778406327289, -0.0897699994287091, -0.155955083059526, 0.143100843640673, -0.0730251350148903, -0.0628009012390303, -0.0145987994211527, -0.00685403642779203, -0.189888919858863, -0.0266682470821618, 0.037740327982847, -0.0111733005981254, -0.117783035656383, -0.0384662808277962, 0, 0.0571584138399484, -0.0694119283885524, -0.0735133828982697, -0.104572803919577, -0.174540558863677, -0.159272329947313, -0.241162056816889, -0.151592882072704, 0.336472236621213, -0.364897923119164, 0.03964483134724, -0.229033345370273, -0.182321556793954, -0.251314428280907, 0.127833371509885, 0.0752830604205483, -0.0685626714372969, -0.0333465799413801, -0.197610249309045, -0.102654154060084, 0.0645385211375711, -0.071686734959413, 0.0268112574506567, -0.11247798342669, -0.116682320080198, 0.143100843640673, -0.0513428079171341, -0.0591888713903304, 0.0285733724440558, -0.2383103438577, -0.0444517625708336, 0.0909717782057271, -0.220542769614153, -0.03650636928728, -0.00735297430525872, -0.136859182717197, -0.0115943327809198, -0.306619273471531, -0.0180556878262053, -0.0133740595252565, -0.847297860387203, -0.0694513863237711, -0.0165293019512109, -0.391671785975829, 0, -0.0750083720980301, -0.246860077931526, -0.351397886837889, -0.23211222129697, 0.131028262406404, 0.0389154162496732, -0.0231224174208542, -0.0749976519277089, -0.0967787628858519, -0.264151575041587, 0.0642791514045653, -0.181470854913642, -0.035350988878931, -0.22314355131421, -0.0585533087515744, -0.170625517030763, -0.117783035656384, -0.223143551314211, -0.271496748030764, -0.231840227823137, -0.0830516576038791, -0.0348753293140129, -0.0905435929019962, -0.039441732051297, -0.091434205959632, -0.0468571586878825, -0.0959639882617438, -0.0117859422190874, -0.0923733201310153, -0.0130171058024899, 0.00475060275859818, -0.0717936920187423, -0.130053128248198, -0.0133312519554591, 0.124674861694305, -0.163954679923878, -0.0845573880280623, -0.132324877296182, 0.0560894666510432, -0.208466836898277, -0.0640702467202106, -0.0487901641694322, -0.0536210914390978, -0.0444517625708345, 0.0861776962410516, -0.00652885888246324, 0, -0.146379535102372, 0.310154928303839, -0.0176995770994006, 0.0115608224010764, 0.100470530363635, 0.162518929497775, 0.264692554227082, -0.113077144618971, -0.120144311842064, 0.059033531584193, 0.00392927813988919, -0.124850202879476, 0.00626961301359508, 0.0298084662978262, -0.118055998671591, -0.131576357788719, -0.00881062968215485, 0.0217399866364056, 0.0184337016888385, 0.079111001652536, -0.038374012881615, -0.305864391534971, -0.167054084663166, 0.126675144754063, -0.0807613566444587, 0.0213911899813182, 0.030459207484709, 0.0540672212702757, 0.145310091817136, -0.025075940768212, -0.274987152970533, -0.232931557680373, -0.103570370042048, -0.141746288743414, 0.00681006216133007, 0.0512932943875506, -0.606135803570316, -0.0969922659873097, -0.0986425361174703, -0.0369435151916848, -0.094288947978324, 0.141650517063027, -1.02961941718116, -0.0822893947565149, -0.0191944472561474, 0.328504066972036, -0.0706175672139531, -1.46633706879343, -0.334639055539552, -0.154681606306414, 0.133531392624523, -0.0668629452291265, 0.0144930073025664, 0.120627987788614, 0.176930708159078, -0.382992252256106, 0.0499410291375364, -0.113944259349218, 0.0173917427118688, -0.168059109873391, -0.0281708769666964, -0.367724780125317, -0.470003629245735, -0.188221278921143, 0, -0.0435712482936399, 0.0165293019512109, 0.255933374137201, 0.0327898228229908, -0.052471051737454, -0.0284494118748899, -0.0729489996461679, 0.112618878108149, -0.0134602190556974, 0.0395764757017059, -0.0756595628451411, -0.492476485097794, 0.0179109265665307, -0.0551320706947163, -0.101096116871369, 0.0107473405956728, 0.092721740050222, -0.034635496662756, -0.00818102901645723, 0.0590490288528001, 0.202236866494896, -0.402992918962775, -0.195516044873487, -0.0569081322718796, -0.0377403279828474, -0.0691272710570754, -0.112417707967825, -0.188742124596877, 0.0428199971829288, 0.0607562094210721, 0.101747438973162, 0.000974184197843009, -0.0126184639592113, -0.0388979479328331, 0.0638081142684905, -0.0234736356127776, 0.0601685214664531, -0.07417986546408, -0.0202436062766465, 0.0964137296680203, -0.0302330106601936, 0.0640072185482552], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } + }, + "derived_columns": { + "lpop_t": "lpop * (year - 2002) / 5" + } + }, + "sim_staggered": { + "data_file": "twfeweights_sim_panel.csv", + "columns": { + "outcome": "outcome", + "unit": "unit", + "time": "period", + "first_treat": "first_treat", + "invariant_cov": "x1", + "varying_cov": "xtv" + }, + "two_period_group": 3, + "attgt_weights": { + "twfe": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "weight": [-0.25, -0.25, 0.375, 0.0625, 0.0625, -0.0625, -0.0625, -0.375, 0.25, 0.25], + "att": [0.0839722781780437, 0, 0.932825921270221, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], + "implied_att": 1.38176729464315 + }, + "overall": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "weight": [0, 0, 0.166666666666667, 0.166666666666667, 0.166666666666667, 0, 0, 0, 0.25, 0.25], + "att": [0.0839722781780437, 0, 0.932825921270221, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], + "implied_att": 1.73623602393453 + }, + "simple": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "weight": [0, 0, 0.2, 0.2, 0.2, 0, 0, 0, 0.2, 0.2], + "att": [0.0839722781780437, 0, 0.932825921270221, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], + "implied_att": 1.76984085043211 + } + }, + "decompose": { + "fwl_nocov": { + "cells": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "att": [0, -0.0839722781780438, 0.848539485953839, 1.76991196956896, 2.84209802207204, 0, 0.0713186971413766, -0.203874498662049, 0.823397571607694, 1.90590552823814], + "weight": [-0.25, -0.25, 0.375, 0.0625, 0.0625, -0.0625, -0.0625, -0.375, 0.25, 0.25], + "ess": [100, 100, 99.4718023394506, 100, 100, 100, 100, 99.4718023394506, 100, 100], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": 1.38176729464315, + "decomposition": 1.38176729464315, + "remainder": 0, + "pre_period_contribution": 0.0929885879714431, + "post_only": 1.28877870667171, + "effective_sample_size": 499.00962938647 + }, + "fwl_cov": { + "cells": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "att": [0, -0.0821486884936649, 34.2972067369434, 1.76608384450794, 2.83842487370002, 0, 0.0730127750279008, 33.2436059932573, 0.823520882264303, 1.90613909549667], + "weight": [-0.250278200442825, -0.250043995598454, 0.375228054332711, 0.0628082321912258, 0.0622859095173414, -0.0622827984811969, -0.06218597232544, -0.375209033152085, 0.249640734110113, 0.250037069848608], + "ess": [99.9941013522363, 99.9944461946957, 0.00609297505217851, 99.9936664953336, 99.9941598816125, 99.9941013522363, 99.9944461946957, 0.00609297505217851, 99.9936664953336, 99.9941598816125], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": 1.38188157642338, + "decomposition": 1.38188157642338, + "remainder": 0, + "pre_period_contribution": -12.4573008473221, + "post_only": 13.8391824237455, + "effective_sample_size": 312.37838960871 + }, + "fwl_gmin1": { + "cells": { + "group": [3, 3, 3, 3, 3, 4, 4, 4, 4, 4], + "time": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5], + "post": [0, 0, 1, 1, 1, 0, 0, 0, 1, 1], + "att": [0.0839722781780438, 0, 0.944867949565343, 1.853884247747, 2.92607030025009, 0.20356034152371, 0.274879038665087, 0, 1.0269579131314, 2.10946586976185], + "weight": [-0.25, -0.25, 0.375, 0.0625, 0.0625, -0.0625, -0.0625, -0.375, 0.25, 0.25], + "ess": [100, 100, 99.4718023394506, 100, 100, 100, 100, 99.4718023394506, 100, 100], + "remainder": [-0.940505910590496, -0.940505910590496, -0.947616432141581, -0.940505910590496, -0.940505910590496, -1.62957956051008, -1.62957956051008, -1.62464805376604, -1.62957956051008, -1.62957956051008] + }, + "estimate": 1.38176729464315, + "decomposition": 1.38628305525382, + "remainder": -0.00451576061067098, + "pre_period_contribution": -0.0508955308063108, + "post_only": 1.43717858606014, + "effective_sample_size": 499.00962938647 + }, + "aipw": { + "cells": { + "group": [3, 3, 3, 4, 4], + "time": [3, 4, 5, 4, 5], + "post": [1, 1, 1, 1, 1], + "att": [0.950890455977506, 1.90549496406924, 3.01528915156681, 1.08514205063569, 2.23370942485274], + "weight": [0.166666666666667, 0.166666666666667, 0.166666666666667, 0.25, 0.25], + "ess": [99.1390613452281, 99.1390613452281, 99.1390613452281, 97.3682491639651, 97.3682491639651] + }, + "estimate": 1.80832529747437, + "decomposition": 1.80832529747437, + "remainder": 0, + "pre_period_contribution": 0, + "post_only": 1.80832529747437, + "effective_sample_size": 491.268276272983 + } + }, + "balance": { + "fwl": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4], + "time": [1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5], + "post": [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1], + "covariate": ["mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv"], + "unweighted_covs_treated": [0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946], + "unweighted_covs_comparison": [0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258], + "unweighted_diff": [-0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121], + "weighted_covs_treated": [0.0322253630566276, 0.61354301308016, 0.0290807000435411, 0.611652284352772, 0.0314456717092465, 0.613125653540162, 0.0304194230673653, 0.611090717337329, 0.0261194212453405, 0.610939920369823, -0.0356287952936007, 0.583175368672645, -0.0455885285626065, 0.579867791877802, -0.0422942050601538, 0.580882621650816, -0.041129084294574, 0.581367403796091, -0.0426164850331386, 0.580717322776911], + "weighted_covs_comparison": [0.128912846595125, 0.667500533623696, 0.127496503049284, 0.666346647112834, -22.2803241752299, -13.6848306944835, 0.129245458534132, 0.667533121252504, 0.128528952295834, 0.667188479625278, 0.128912846595125, 0.667500533623696, 0.127496503049284, 0.666346647112834, -22.2803241752299, -13.6848306944835, 0.129245458534132, 0.667533121252504, 0.128528952295834, 0.667188479625278], + "weighted_diff": [-0.0966874835384971, -0.0539575205435364, -0.0984158030057429, -0.0546943627600621, 22.3117698469391, 14.2979563480237, -0.0988260354667672, -0.0564424039151755, -0.102409531050493, -0.0562485592554547, -0.164541641888725, -0.0843251649510506, -0.17308503161189, -0.0864788552350323, 22.2380299701697, 14.2657133161343, -0.170374542828706, -0.0861657174564133, -0.171145437328972, -0.0864711568483669], + "sd": [1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979], + "unweighted_log_ratio_sd_diff": [-0.0785342770763817, -0.0976886323541, -0.0785342770763817, -0.0976886323541, -0.0785342770763817, -0.0976886323541, -0.0785342770763817, -0.0976886323541, -0.0785342770763817, -0.0976886323541, 0.0147899208029592, -0.000406987372867373, 0.0147899208029592, -0.000406987372867373, 0.0147899208029592, -0.000406987372867373, 0.0147899208029592, -0.000406987372867373, 0.0147899208029592, -0.000406987372867373], + "weighted_log_ratio_sd_diff": [-0.078923039541646, -0.0981028673500817, -0.0787851460177666, -0.0992789443199351, -4.79800027862989, -5.26202924360934, -0.0788460372935362, -0.0965119215695593, -0.0755098089684916, -0.104025654316807, 0.027732869455797, 0.0160140941570539, 0.0107487270326319, -0.00359213115639778, -4.70526255947979, -5.16554315809729, 0.0155949542480163, 3.59691177780341e-05, 0.0154199797107926, 0.000125226527590172], + "unweighted_frac_treated_extreme": [0.05, 0.02, 0.05, 0.02, 0.05, 0.02, 0.05, 0.02, 0.05, 0.02, 0.12, 0.08, 0.12, 0.08, 0.12, 0.08, 0.12, 0.08, 0.12, 0.08], + "weighted_frac_treated_extreme": [0.05, 0.02, 0.04, 0.02, 0, 0, 0.05, 0.03, 0.04, 0.02, 0.11, 0.08, 0.1, 0.09, 0, 0, 0.11, 0.08, 0.11, 0.09] + }, + "summary": { + "unweighted_treat": [-0.00530837076689503, 0.597037457313671], + "unweighted_untreat": [0.128201005218175, 0.666936103061258], + "unweighted_diff": [-0.13350937598507, -0.0698986457475872], + "weighted_treat": [-0.00558643553339065, 0.596830265573273], + "weighted_untreat": [-8.27967753493962, -4.71798406790115], + "weighted_diff": [8.27409109940623, 5.31481433347443], + "sd": [1.02583786880053, 0.576529078265404], + "unweighted_log_ratio_sd_diff": [-0.0319022468238235, -0.0490791536243857], + "weighted_log_ratio_sd_diff": [-1.80225098421397, -1.98696177996179], + "unweighted_frac_treated_extreme": [0.0849774462771105, 0.0499806682375233], + "weighted_frac_treated_extreme": [0.0605964064257143, 0.0456045601712674], + "covariate": ["mean_x1", "mean_xtv"] + } + }, + "aipw": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 4, 4, 4, 4], + "time": [3, 3, 4, 4, 5, 5, 4, 4, 5, 5], + "post": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "covariate": ["mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv"], + "unweighted_covs_treated": [0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946], + "unweighted_covs_comparison": [0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258, 0.128201005218175, 0.666936103061258], + "unweighted_diff": [-0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.0967916911751303, -0.0538294046611455, -0.170274412478823, -0.0859886099713121, -0.170274412478823, -0.0859886099713121], + "weighted_covs_treated": [0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, 0.031409314043045, 0.613106698400113, -0.0420734072606479, 0.580947493089946, -0.0420734072606479, 0.580947493089946], + "weighted_covs_comparison": [0.031409314043045, 0.615961633357426, 0.031409314043045, 0.615961633357426, 0.031409314043045, 0.615961633357426, -0.0420734072606479, 0.57732449107713, -0.0420734072606479, 0.57732449107713], + "weighted_diff": [0, -0.0028549349573127, 0, -0.0028549349573127, 0, -0.0028549349573127, 2.77555756156289e-17, 0.00362300201281618, 2.77555756156289e-17, 0.00362300201281618], + "sd": [1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.00268454126303, 0.563205053222823, 1.04902105522077, 0.589870286172979, 1.04902105522077, 0.589870286172979] + }, + "summary": { + "unweighted_treat": [-0.00533204660880145, 0.59702709574503], + "unweighted_untreat": [0.128201005218175, 0.666936103061258], + "unweighted_diff": [-0.133533051826977, -0.0699090073162288], + "weighted_treat": [-0.00533204660880145, 0.59702709574503], + "weighted_untreat": [-0.00533204660880146, 0.596643062217278], + "weighted_diff": [1.38777878078145e-17, 0.000384033527751737], + "sd": [1.0258527982419, 0.576537669697901], + "unweighted_log_ratio_sd_diff": [0, 0], + "weighted_log_ratio_sd_diff": [0, 0], + "unweighted_frac_treated_extreme": [0, 0], + "weighted_frac_treated_extreme": [0, 0], + "covariate": ["mean_x1", "mean_xtv"] + } + } + }, + "two_period": { + "reg": { + "estimate": 0.933749536861636, + "ess": 99.9926610233913, + "weights": [1.00722890988084, 0.992990992262401, 0.985696926864851, 1.00428546538914, 1.00255445781008, 0.99320823416596, 0.999615523785608, 1.00343083006454, 1.00137356889656, 0.986745204378563, 0.995094957332289, 0.986870840632856, 1.00528820819911, 0.998835332251396, 0.99700861697196, 0.997936725463451, 0.998846629895534, 0.99656967557782, 1.00045816004973, 0.990771166474457, 0.997338926459884, 0.998846092341373, 0.995772857502799, 1.01166869648943, 0.993003662361777, 1.00600474847494, 1.00405785167947, 1.00353809034622, 0.995126225338609, 1.00043085640342, 0.998206308876953, 0.996320942216609, 1.00237756422585, 0.985442690419153, 0.989720250034912, 0.995867683639298, 0.99111809890224, 1.00019084884885, 0.990912512219895, 1.00911662153634, 1.0192714498591, 1.01698591482046, 1.00242798608037, 0.991647428774127, 1.00134810289326, 1.00443878939999, 0.996778530830711, 1.00669024403109, 1.00001152211208, 1.01998818237323, 0.99264596683436, 0.9952948133176, 1.01360504654043, 1.009933572531, 1.01298877137443, 1.00106309713833, 0.99368962673189, 0.994645710006753, 1.00603342986714, 0.995879858693205, 1.01601455811524, 1.02010680280297, 0.989784340649968, 1.00825224525215, 1.00677285251964, 1.00871986746097, 1.00390290828464, 0.993629249244157, 1.00167286392168, 1.01886807947052, 1.01099154001983, 1.00201217904192, 0.99836526676243, 1.01002660972161, 0.996287759273861, 1.00250415790818, 0.983606011950821, 1.00384205736887, 0.991605834088229, 1.00537797220874, 1.01272476427711, 0.99235753997329, 0.987933977679291, 1.000650667459, 0.984180988085733, 0.996029922902563, 0.997939009434197, 0.990121709237807, 0.991953055467323, 1.00235884266265, 1.00125277134316, 0.992473987574461, 0.992917670060727, 0.998942314836784, 1.00485857501194, 1.00287939258367, 1.00418948560417, 0.985443467362189, 0.985970178418021, 1.01723852508477, 0.997466706557304, 0.992423956214259, 1.00991905980572, 0.992291387074752, 0.994820831943633, 0.989354780777259, 1.00319011306147, 1.00827468557464, 1.00431722998307, 0.996353080619044, 1.00439019192514, 0.999040966656546, 1.00231586960461, 0.978372337508447, 0.995019298342224, 0.992937322106751, 0.993590737901513, 1.00475685538413, 1.00449017738004, 1.01172939172658, 0.996227103707377, 0.998538789522835, 0.995737537791419, 0.99411674103665, 1.00138684721952, 1.00931125598526, 0.99549586869746, 1.00710500069773, 1.01347249983801, 0.996627686726185, 1.00244252373843, 1.00388526279484, 0.996225099533706, 1.01622034867275, 0.992126287936598, 0.998427079293577, 0.997025997515504, 0.994683358673275, 0.994260944805646, 1.01294005227182, 1.01510040197259, 1.00867105634545, 0.999364111810118, 0.988370930721448, 1.00754081330585, 1.01108908526376, 1.00082929965702, 0.999652563328744, 0.99115023644786, 0.993797952889325, 1.00062317593941, 0.996370067258762, 1.00862548801356, 0.992934433889504, 1.00625697060545, 1.01325255367648, 1.00669485794986, 0.994504199351021, 0.998979328323581, 0.989317777254636, 1.0079062968392, 1.00142985808414, 1.00927426598237, 0.988654526955168, 0.989280380585935, 1.00609361546911, 0.997618744628682, 1.00320239737719, 1.00752528142108, 1.00990809476446, 1.00300322465167, 0.991801531478728, 0.982098203337734, 0.993156779738883, 0.99999216518783, 1.01701536391709, 1.00655564623742, 0.987239057302495, 0.997613724292227, 0.996893085121885, 0.998388510818664, 1.00686703463208, 0.989947899004373, 0.99691959100035, 0.992555666528202, 1.02749634237056, 0.990975757502908, 0.998821219235584, 1.00181547622219, 0.982006002279984, 1.00349537276095, 1.01340976418696, 1.00161767714608, 0.998384149967381, 1.01077308854491, 0.993352591748675, 1.00138206112585, 1.00203727492767, 0.990541643636813, 0.988514062376352], + "dy": [-0.402033610301275, 2.18544275172545, 0.407877616958751, 1.8390018429467, 0.342089980783171, 5.17283289194947, 1.77541144069014, 0.177916426818689, 0.988193170415602, -1.73303800372523, -0.455268584403767, 0.627245807499127, 0.95163643365076, 0.729499027127282, -0.716548210368217, 0.989179554960285, 0.615290027522918, -0.928173862369419, -3.12066260941806, 0.254525318474001, 2.26949886412902, 0.829906104908926, 0.839722075215133, 0.402827764479532, 2.615721647119, 2.56884939538828, 0.40984832983609, 0.916597426324454, 3.56815993643643, 4.11857018395454, -0.583954358728381, 0.629543252397921, -2.59864890963595, -1.50575253730084, 0.10854369311674, 0.339066188455571, 0.840969127612396, 1.49640162269824, 2.15698629580388, 0.710710905149634, -0.780658854871792, 0.246639873748455, -2.99165681128261, 2.21630573154794, 1.36462660688514, 3.91230934269807, 0.750821589315842, 1.3797955671716, 1.29873425008199, -0.0142518982330498, 0.942704674378142, -0.00295039223517701, 1.26619389096853, 2.07005386994108, 1.30556335439981, 0.957684657053287, 2.07603562299318, 3.09870706931771, -1.07466886872445, 1.72619332970284, 0.185899746718614, 0.879844294714601, -0.110555708727767, 0.867499070153156, 1.97688576587969, 2.11054101983651, -0.213279861129251, 0.590859861669465, 0.286669554875457, -1.33313130715994, 0.166213410436903, -0.981498645950401, -1.08091042671938, -2.43836942530926, -0.476981977649933, 0.165773714712555, 1.34280270525293, 2.29324397361844, -2.48270705209481, -1.77399367825062, 0.528226838838877, 1.79264254406572, 1.15140499752099, 2.42338563275737, 0.813914692972622, 3.9331810215408, 0.638302157203735, 0.954304368656401, 0.44612105486203, 1.09255026112807, 1.66177432848481, 1.2244211095895, -0.307344980789505, 0.362311341134403, -0.707302108346803, -0.144104822654965, 2.80967986565376, 0.886724301741435, -0.437691146279499, 0.227891475847955, 4.03744118286034, 2.45748912778531, 0.144254311972021, 2.61650674781118, 4.31326476995183, 1.48977229748571, 2.34613355467357, 1.70052061441654, 3.22056170230932, 1.78075346385693, -0.711428102809615, 1.34730204516023, 1.03029253615151, 0.419549571357165, 1.58776493288537, 0.995197584563667, 0.642274699864077, 2.07636006524544, 2.38487012186388, 1.23008817197199, 1.38552304464906, -0.660629532295937, 2.90020370216734, 2.14880989578056, 0.0397416585750139, 3.38848652983915, 1.5391831069985, 3.45379719174161, 0.398418170016988, 2.92591755647753, 0.269001120074444, 3.49568149729268, 1.10053690527001, 0.534365065625511, 2.45681592071403, 1.59579092024735, 0.819893378575548, 2.81625032274877, -0.00324530506939347, 0.605258529293289, 3.85479849191654, 0.583541472525013, 2.80499048092448, 3.43673863609605, 2.53588037120317, 1.05364560288094, 1.74797954925627, 1.12861746747233, 2.84125481556006, 3.12911840005426, 0.302617858060385, 2.05438327553933, 0.414404978188778, 1.89216322378059, 1.13139413791383, 2.00133216270611, 0.643782885192826, 1.88591067421528, 0.252206326893992, 0.687547110267291, 1.95059516009538, 0.154987971181278, -1.84401576567585, 1.42142999538519, 1.45740057865859, 0.642256402067309, 2.85687382154418, 2.56544873039477, 1.55960416827183, 5.34308795625776, -0.487166716367998, 1.11969723983059, 1.31495808447936, 1.95142653439584, 3.50775782707169, 1.04468240755541, 2.15013315602819, 2.36947457467637, 1.91237343735871, 0.613733460400012, 4.06876671513254, 3.00698872518514, 2.43255060792113, 0.590220612096163, 0.790617682152215, 0.3088784929935, -0.390557110671398, 0.783542381656206, 2.54825966725677, 0.531623874159247, 0.754254870486599, 2.26979312680552, 0.882020570135368, 3.67733208815144, 1.18041126492999, 1.67479917330522, 1.13098167264533, 3.93443859164881, 0.545035532418525, -0.835817441782716], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + }, + "aipw": { + "estimate": 0.932908107897949, + "ess": 99.999180206262, + "weights": [0.999672729706382, 0.998021972909027, 1.00246030815386, 0.998513055504986, 1.00585412572353, 0.997051177858498, 0.994461321822849, 0.996136791680998, 0.99802322685533, 1.00130465103858, 1.0036020329346, 1.00209547702384, 1.00124115667369, 0.996393987795192, 0.998049799389463, 1.00262259770517, 1.00030309128241, 1.00064739548438, 1.00291569113999, 1.00435106542457, 1.0011434294141, 1.00188564912852, 1.00179620123262, 0.995509793569397, 1.00003833654497, 1.00001014848312, 1.00212284054395, 1.00181323882764, 1.00114490591003, 0.999410260939198, 1.00368717706179, 1.00000109453662, 0.998485491222804, 1.00698466777593, 0.996625230168786, 1.00352109899527, 0.995024150944156, 0.998998191251484, 1.00554621969829, 0.997495901024524, 0.999565102450949, 0.99382696946915, 0.993461661221226, 0.999828271614111, 1.00045584463728, 0.996827971795766, 1.00081229366678, 0.998565528198841, 1.00389512128434, 0.995518787588971, 1.00440712427909, 0.999875209026417, 1.0032200133693, 0.999900835948388, 0.997523979887541, 1.00025940775218, 1.00175413984952, 1.0043515707889, 0.999784272657803, 0.998278347015222, 1.00199659550982, 0.998870210394531, 1.00435304482845, 1.00013634872456, 0.996946252359397, 0.996194426606418, 0.995532736428361, 0.998699972048666, 0.994346574614814, 0.997342357218583, 1.00141195618414, 0.998554195649132, 0.995582390998808, 1.00104190020894, 0.996355646424078, 0.999910936252083, 1.00118919151784, 0.994777646401869, 1.00165288081142, 1.00136294435556, 1.00030533982388, 0.999923527203744, 0.999932083612262, 1.00055286791905, 1.00310726818895, 1.0008836585536, 1.00106078377252, 1.00232921480015, 1.00148651721726, 0.999986839444585, 1.00007595480953, 1.0026103987571, 0.999783262359765, 1.00559848648777, 0.998470814142084, 0.996007891476203, 1.00066584402688, 1.00217882663762, 1.00022113959277, 1.00148293775449, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "dy": [-0.402033610301275, 2.18544275172545, 0.407877616958751, 1.8390018429467, 0.342089980783171, 5.17283289194947, 1.77541144069014, 0.177916426818689, 0.988193170415602, -1.73303800372523, -0.455268584403767, 0.627245807499127, 0.95163643365076, 0.729499027127282, -0.716548210368217, 0.989179554960285, 0.615290027522918, -0.928173862369419, -3.12066260941806, 0.254525318474001, 2.26949886412902, 0.829906104908926, 0.839722075215133, 0.402827764479532, 2.615721647119, 2.56884939538828, 0.40984832983609, 0.916597426324454, 3.56815993643643, 4.11857018395454, -0.583954358728381, 0.629543252397921, -2.59864890963595, -1.50575253730084, 0.10854369311674, 0.339066188455571, 0.840969127612396, 1.49640162269824, 2.15698629580388, 0.710710905149634, -0.780658854871792, 0.246639873748455, -2.99165681128261, 2.21630573154794, 1.36462660688514, 3.91230934269807, 0.750821589315842, 1.3797955671716, 1.29873425008199, -0.0142518982330498, 0.942704674378142, -0.00295039223517701, 1.26619389096853, 2.07005386994108, 1.30556335439981, 0.957684657053287, 2.07603562299318, 3.09870706931771, -1.07466886872445, 1.72619332970284, 0.185899746718614, 0.879844294714601, -0.110555708727767, 0.867499070153156, 1.97688576587969, 2.11054101983651, -0.213279861129251, 0.590859861669465, 0.286669554875457, -1.33313130715994, 0.166213410436903, -0.981498645950401, -1.08091042671938, -2.43836942530926, -0.476981977649933, 0.165773714712555, 1.34280270525293, 2.29324397361844, -2.48270705209481, -1.77399367825062, 0.528226838838877, 1.79264254406572, 1.15140499752099, 2.42338563275737, 0.813914692972622, 3.9331810215408, 0.638302157203735, 0.954304368656401, 0.44612105486203, 1.09255026112807, 1.66177432848481, 1.2244211095895, -0.307344980789505, 0.362311341134403, -0.707302108346803, -0.144104822654965, 2.80967986565376, 0.886724301741435, -0.437691146279499, 0.227891475847955, 4.03744118286034, 2.45748912778531, 0.144254311972021, 2.61650674781118, 4.31326476995183, 1.48977229748571, 2.34613355467357, 1.70052061441654, 3.22056170230932, 1.78075346385693, -0.711428102809615, 1.34730204516023, 1.03029253615151, 0.419549571357165, 1.58776493288537, 0.995197584563667, 0.642274699864077, 2.07636006524544, 2.38487012186388, 1.23008817197199, 1.38552304464906, -0.660629532295937, 2.90020370216734, 2.14880989578056, 0.0397416585750139, 3.38848652983915, 1.5391831069985, 3.45379719174161, 0.398418170016988, 2.92591755647753, 0.269001120074444, 3.49568149729268, 1.10053690527001, 0.534365065625511, 2.45681592071403, 1.59579092024735, 0.819893378575548, 2.81625032274877, -0.00324530506939347, 0.605258529293289, 3.85479849191654, 0.583541472525013, 2.80499048092448, 3.43673863609605, 2.53588037120317, 1.05364560288094, 1.74797954925627, 1.12861746747233, 2.84125481556006, 3.12911840005426, 0.302617858060385, 2.05438327553933, 0.414404978188778, 1.89216322378059, 1.13139413791383, 2.00133216270611, 0.643782885192826, 1.88591067421528, 0.252206326893992, 0.687547110267291, 1.95059516009538, 0.154987971181278, -1.84401576567585, 1.42142999538519, 1.45740057865859, 0.642256402067309, 2.85687382154418, 2.56544873039477, 1.55960416827183, 5.34308795625776, -0.487166716367998, 1.11969723983059, 1.31495808447936, 1.95142653439584, 3.50775782707169, 1.04468240755541, 2.15013315602819, 2.36947457467637, 1.91237343735871, 0.613733460400012, 4.06876671513254, 3.00698872518514, 2.43255060792113, 0.590220612096163, 0.790617682152215, 0.3088784929935, -0.390557110671398, 0.783542381656206, 2.54825966725677, 0.531623874159247, 0.754254870486599, 2.26979312680552, 0.882020570135368, 3.67733208815144, 1.18041126492999, 1.67479917330522, 1.13098167264533, 3.93443859164881, 0.545035532418525, -0.835817441782716], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + } + } + }, + "unbalanced_cohorts": { + "data_file": "twfeweights_unbalanced_panel.csv", + "columns": { + "outcome": "outcome", + "unit": "unit", + "time": "period", + "first_treat": "first_treat", + "invariant_cov": "x1", + "varying_cov": "xtv" + }, + "two_period_group": 3, + "attgt_weights": { + "twfe": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "weight": [-0.265151515151515, -0.265151515151515, 0.212121212121212, 0.212121212121212, 0.053030303030303, 0.053030303030303, -0.0378787878787879, -0.0378787878787879, -0.196969696969697, -0.196969696969697, 0.234848484848485, 0.234848484848485], + "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914813, 0, 1.04128383798297, 2.31517239203707], + "implied_att": 1.88846922090824 + }, + "overall": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "weight": [0, 0, 0.134615384615385, 0.134615384615385, 0.134615384615385, 0.134615384615385, 0, 0, 0, 0, 0.230769230769231, 0.230769230769231], + "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914813, 0, 1.04128383798297, 2.31517239203707], + "implied_att": 2.25856121537518 + }, + "simple": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "weight": [0, 0, 0.175, 0.175, 0.175, 0.175, 0, 0, 0, 0, 0.15, 0.15], + "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914813, 0, 1.04128383798297, 2.31517239203707], + "implied_att": 2.43266114548473 + } + }, + "decompose": { + "fwl_nocov": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "att": [0, -0.14625813390341, 1.14471439240315, 2.1890605959757, 3.06376746663026, 4.04138335784424, 0, -0.0481726004625502, 0.331153365133827, 0.149892455219012, 1.19117629320198, 2.46506484725609], + "weight": [-0.265151515151515, -0.265151515151515, 0.212121212121213, 0.212121212121213, 0.053030303030302, 0.053030303030302, -0.0378787878787875, -0.0378787878787875, -0.196969696969696, -0.196969696969696, 0.234848484848484, 0.234848484848484], + "ess": [120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": 1.88846922090823, + "decomposition": 1.88846922090823, + "remainder": 0, + "pre_period_contribution": -0.0541461639412249, + "post_only": 1.94261538484946, + "effective_sample_size": 719.999999999998 + }, + "fwl_cov": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "att": [0, -0.146200722768517, 1.17426164200888, 2.18125058212415, 3.08489273485064, 4.02603401811628, 0, -0.0607908732286701, 0.361790510905905, 0.134790399259838, 1.18902450338214, 2.46470121594505], + "weight": [-0.265232105650193, -0.265193408359637, 0.212690896337156, 0.211910908562023, 0.0525407450858316, 0.0532829640248196, -0.0383900377379283, -0.0374892499005717, -0.197107012378148, -0.196588185973519, 0.235333389228316, 0.234241096761852], + "ess": [119.857036307608, 119.848224124526, 83.99514247147, 85.9094063814382, 119.861183035227, 119.861142580033, 119.857036307608, 119.848224124526, 83.99514247147, 85.9094063814382, 119.861183035227, 119.861142580033], + "remainder": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "estimate": 1.88897945007637, + "decomposition": 1.88897945007637, + "remainder": 0, + "pre_period_contribution": -0.0567591745747959, + "post_only": 1.94573862465116, + "effective_sample_size": 630.228235498893 + }, + "fwl_gmin1": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5], + "time": [1, 2, 3, 4, 5, 6, 1, 2, 3, 4, 5, 6], + "post": [0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1], + "att": [0.14625813390341, 0, 1.29097252630656, 2.33531872987912, 3.21002560053367, 4.18764149174765, -0.149892455219016, -0.198065055681567, 0.181260909914814, 0, 1.04128383798297, 2.31517239203707], + "weight": [-0.265151515151515, -0.265151515151515, 0.212121212121213, 0.212121212121213, 0.053030303030302, 0.053030303030302, -0.0378787878787875, -0.0378787878787875, -0.196969696969696, -0.196969696969696, 0.234848484848484, 0.234848484848484], + "ess": [120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120, 120], + "remainder": [-0.945688714374791, -0.945688714374791, -0.945688714374792, -0.945688714374792, -0.945688714374791, -0.945688714374791, -1.88599400101312, -1.88599400101312, -1.88599400101312, -1.88599400101312, -1.88599400101312, -1.88599400101312] + }, + "estimate": 1.88846922090823, + "decomposition": 1.88846922090823, + "remainder": -1.52655665885959e-16, + "pre_period_contribution": -0.0613032635598606, + "post_only": 1.94977248446809, + "effective_sample_size": 719.999999999998 + }, + "aipw": { + "cells": { + "group": [3, 3, 3, 3, 5, 5], + "time": [3, 4, 5, 6, 5, 6], + "post": [1, 1, 1, 1, 1, 1], + "att": [1.28404655298829, 2.32056347203679, 3.18519253119276, 4.15837159652383, 0.966820926227897, 2.20786930029771], + "weight": [0.134615384615385, 0.134615384615385, 0.134615384615385, 0.134615384615385, 0.230769230769231, 0.230769230769231], + "ess": [119.903588727274, 119.903588727274, 119.903588727274, 119.903588727274, 115.072096311797, 115.072096311797] + }, + "estimate": 2.20641349591344, + "decomposition": 2.20641349591344, + "remainder": 0, + "pre_period_contribution": 0, + "post_only": 2.20641349591344, + "effective_sample_size": 706.0420149054 + } + }, + "balance": { + "fwl": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5], + "time": [1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6], + "post": [0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1], + "covariate": ["mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv"], + "unweighted_covs_treated": [-0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615], + "unweighted_covs_comparison": [-0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237], + "unweighted_diff": [0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778], + "weighted_covs_treated": [-0.0552006329079015, 0.638635766785036, -0.0535601307184772, 0.640404316629236, -0.0544923109469719, 0.6401601593518, -0.0577693387390503, 0.637312816179378, -0.032929972143668, 0.648304910559683, -0.0616076261294278, 0.637080053522665, 0.0835621488155271, 0.737738482174229, 0.0945036039710708, 0.744157273744997, 0.108279525337989, 0.748815310739362, 0.103457592255963, 0.746870083930084, 0.104726058647783, 0.747708304746342, 0.101546993276873, 0.745734043929024], + "weighted_covs_comparison": [-0.0820211156139367, 0.693680674197995, -0.0796214274629394, 0.694352196990227, -0.0689935730028542, 0.71126680230441, -0.056084159820465, 0.688019746753786, -0.0799713013479677, 0.694299297567012, -0.0836355509702227, 0.693116732796288, -0.0820211156139367, 0.693680674197995, -0.0796214274629394, 0.694352196990227, -0.0689935730028542, 0.71126680230441, -0.056084159820465, 0.688019746753786, -0.0799713013479677, 0.694299297567012, -0.0836355509702227, 0.693116732796288], + "weighted_diff": [0.0268204827060351, -0.0550449074129595, 0.0260612967444623, -0.0539478803609912, 0.0145012620558822, -0.0711066429526097, -0.00168517891858527, -0.0507069305744083, 0.0470413292042998, -0.0459943870073296, 0.0220279248407949, -0.0560366792736232, 0.165583264429464, 0.0440578079762343, 0.17412503143401, 0.0498050767547699, 0.177273098340843, 0.0375485084349517, 0.159541752076428, 0.0588503371762982, 0.184697359995751, 0.0534090071793297, 0.185182544247096, 0.0526173111327358], + "sd": [0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283], + "unweighted_log_ratio_sd_diff": [-0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.332695249118714, -0.393312159582347, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277, -0.313227965548328, -0.327027963751277], + "weighted_log_ratio_sd_diff": [-0.327379355480894, -0.388856622667763, -0.332728063219466, -0.393001836482863, -0.482246153471369, -0.760575572777059, -0.444382241712145, -0.729047181913841, -0.341186304526637, -0.365744546430006, -0.294845966896052, -0.373086629641763, -0.277042097435864, -0.27985283207957, -0.331236252179694, -0.345749079434454, -0.457886921184364, -0.693111339953135, -0.430872341074444, -0.671839969125508, -0.316957716605812, -0.335463807510608, -0.312906075652989, -0.333392440708849], + "unweighted_frac_treated_extreme": [0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0285714285714285, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333, 0.133333333333333, 0.0333333333333333], + "weighted_frac_treated_extreme": [0.0285714285714285, 0.0142857142857143, 0.0142857142857142, 0.0285714285714285, 0.0142857142857142, 0.0142857142857143, 0.0142857142857142, 0.0142857142857143, 0.0428571428571429, 0.0285714285714285, 0.0428571428571428, 0.0285714285714285, 0.133333333333333, 0.05, 0.133333333333333, 0.0333333333333333, 0.05, 0, 0.05, 0, 0.0833333333333333, 0.0333333333333333, 0.1, 0.0333333333333333] + }, + "summary": { + "unweighted_treat": [0.0202808028752963, 0.690196397646397], + "unweighted_untreat": [-0.081027435542576, 0.693933252829235], + "unweighted_diff": [0.101308238417872, -0.00373685518283795], + "weighted_treat": [0.0195872436833426, 0.689860002475752], + "weighted_untreat": [-0.0736280525335835, 0.696237410214999], + "weighted_diff": [0.0932152962169262, -0.00637740773924623], + "sd": [0.904821714009258, 0.497941950253664], + "unweighted_log_ratio_sd_diff": [-0.323553909442524, -0.3621867923957], + "weighted_log_ratio_sd_diff": [-0.378261456962234, -0.512396249204556], + "unweighted_frac_treated_extreme": [0.0701874388083532, 0.030807497552334], + "weighted_frac_treated_extreme": [0.0536362673342276, 0.0247417574823458], + "covariate": ["mean_x1", "mean_xtv"] + } + }, + "aipw": { + "cells": { + "group": [3, 3, 3, 3, 3, 3, 3, 3, 5, 5, 5, 5], + "time": [3, 3, 4, 4, 5, 5, 6, 6, 5, 5, 6, 6], + "post": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "covariate": ["mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv", "mean_x1", "mean_xtv"], + "unweighted_covs_treated": [-0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615], + "unweighted_covs_comparison": [-0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237, -0.0810274355425762, 0.693933252829237], + "unweighted_diff": [0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.0257840899049139, -0.0550598981348612, 0.186619379653897, 0.0542370174413778, 0.186619379653897, 0.0542370174413778], + "weighted_covs_treated": [-0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, -0.0552433456376623, 0.638873354694376, 0.105591944111321, 0.748170270270615, 0.105591944111321, 0.748170270270615], + "weighted_covs_comparison": [-0.0552433456376623, 0.707144787976769, -0.0552433456376623, 0.707144787976769, -0.0552433456376623, 0.707144787976769, -0.0552433456376623, 0.707144787976769, 0.105591944111321, 0.789549339556244, 0.105591944111321, 0.789549339556244], + "weighted_diff": [-1.38777878078145e-17, -0.0682714332823932, -1.38777878078145e-17, -0.0682714332823932, -1.38777878078145e-17, -0.0682714332823932, -1.38777878078145e-17, -0.0682714332823932, -8.32667268468867e-17, -0.041379069285629, -8.32667268468867e-17, -0.041379069285629], + "sd": [0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.890351861072436, 0.486463145100192, 0.921166680379138, 0.510908266153283, 0.921166680379138, 0.510908266153283] + }, + "summary": { + "unweighted_treat": [0.0189883265541762, 0.689318084960332], + "unweighted_untreat": [-0.0810274355425762, 0.693933252829237], + "unweighted_diff": [0.100015762096752, -0.00461516786890477], + "weighted_treat": [0.0189883265541762, 0.689318084960332], + "weighted_untreat": [0.0189883265541762, 0.745177657936526], + "weighted_diff": [-4.5903451979694e-17, -0.0558595729761943], + "sd": [0.904574085367837, 0.497745508663157], + "unweighted_log_ratio_sd_diff": [0, 0], + "weighted_log_ratio_sd_diff": [0, 0], + "unweighted_frac_treated_extreme": [0, 0], + "weighted_frac_treated_extreme": [0, 0], + "covariate": ["mean_x1", "mean_xtv"] + } + } + }, + "two_period": { + "reg": { + "estimate": 1.28957044213691, + "ess": 119.995700148386, + "weights": [1.00925477055982, 0.998723185675059, 0.996634725503275, 1.0062962595469, 1.0116846563981, 1.00452028093292, 1.00771486087647, 1.00204953103351, 1.0081898745094, 0.993138351267652, 1.00064950601217, 1.00958832842553, 1.00708633443042, 1.00143658864871, 0.999489519214752, 1.00911008327078, 1.00168297069056, 0.983050385189857, 1.00668374658622, 0.994367888846623, 1.00306411999543, 1.00581032890592, 0.995834544735198, 1.00135304942198, 0.997982911644572, 1.00549542164811, 0.992021816202564, 0.99946715234167, 0.994504899838761, 1.00336245204938, 0.99810636855178, 1.0042481533285, 1.00202880076963, 0.99377550716937, 1.00083681709583, 0.993840646573631, 0.995025443722193, 0.996441015885937, 1.00837717223639, 1.00287990494642, 1.00165231794312, 0.996029676232405, 0.990238585162026, 0.995938530509862, 0.998372849179898, 1.00217444576362, 1.0017515355913, 0.997885202550526, 1.00957796620665, 0.99454446944383, 1.00781508610486, 0.989676661130244, 1.00195850082524, 1.00228415784846, 1.0044654432561, 1.01551328546879, 0.993101453522634, 1.00042397201405, 1.00324727474383, 0.996107026201761, 1.00279636023904, 0.999681571328118, 1.00552645669481, 1.00510148957087, 0.991265296747816, 0.999928753044178, 0.994735345537259, 0.998610758079682, 0.999659826801045, 1.00695181771713, 1.00087111390321, 0.99239696826285, 0.997596950451456, 1.00167680363745, 0.998186813694359, 1.00314547233257, 0.989000472345918, 0.989810225580615, 1.00068860595832, 1.00045926654644, 1.00185508614782, 0.997098887679209, 0.995092885869272, 1.00882365715098, 0.995943321465413, 0.995345394281881, 1.00242170714711, 1.01131485449591, 1.01546992314392, 1.00473931229751, 1.00454884206903, 1.00120579562608, 0.991933253970624, 0.992552423967053, 1.00103232626908, 0.996073666367965, 1.00606469413823, 0.993935153780701, 0.993405482710502, 1.00576004097157, 1.00404133753852, 0.990720622466242, 0.9895886575647, 0.990789224140744, 0.998229047704058, 0.993249994137338, 0.997704430441061, 1.00139407975238, 0.997384073985491, 0.998716973738327, 0.996509691218583, 0.994121921942389, 1.00069492728128, 0.99361118386715, 1.01106122493825, 0.999614467420359, 0.99719516770885, 1.00761213299192, 1.00044650582276, 0.996040442935408, 0.995776410452928, 1.00014559518799, 1.00163036922118, 0.998107414758181, 0.998929265083554, 1.00394146389334, 0.994663562354314, 0.997264291076991, 0.999722078445559, 1.00127725705156, 0.999660594616231, 0.997495701681234, 0.999221017580553, 0.999202469015296, 1.00137763212731, 1.00356440217487, 0.998483204711311, 0.999264607253516, 1.00003520861784, 1.00231366955048, 0.996020219541034, 1.00107946453243, 1.00122012240469, 0.995993541722388, 0.9957999534808, 1.00358672328067, 1.00377596409354, 1.00283713463745, 0.998897288822817, 1.00140206334268, 0.994859786413371, 0.999617444016502, 1.00558977187743, 1.00151742123166, 0.999440356527955, 1.00348353086931, 1.00103531753353, 0.994836036333979, 0.996484335173441, 1.00690040871755, 1.00252882679405, 0.996057521027822, 1.00074181334411, 0.999624789407533, 0.997053878174417, 0.999466692467566, 0.998408862902966, 0.998885822158993, 0.996357520999377, 0.999797669251639, 0.99870100074395, 1.00039444814759, 1.00012122673967, 1.00333196253064, 0.997811178378975, 1.00012878664617, 1.00344301864085, 0.998071892714265, 1.00306218854884, 1.00194832910166, 0.998881145839971, 1.00128524589817, 0.999049662363083, 0.995286955499897, 1.00136819116663, 0.9990008180002, 1.00499457305456, 0.999427617457224, 1.00038272399958, 1.00793253859212], + "dy": [1.38026515156733, 0.408266791145164, -0.339431469645495, -0.682691992758838, -1.82871929455127, 1.14929021958467, 0.406093652733488, 1.39592916302462, 3.44235885109425, -0.820263245202869, -0.0318323352249712, 1.85077537805184, 0.662440201648147, -0.687203070639989, 1.81011852855793, -0.139481071156265, 2.4168721220432, 0.379022022235292, -0.914287504013753, 0.944043804231505, -0.215693960694542, 1.41072414986002, 1.36325294550578, -1.46123322504876, 0.715429531063086, 0.26309981815957, -0.358751783679667, 1.84249495658836, 1.73947344841319, -1.75329820036687, -0.379579864813726, -3.50097882222108, -1.05188660915389, 0.2536693555151, -0.971711160276842, -1.19521084045457, -2.01434628951932, 3.06082555979665, 0.838004826699251, 2.54024677095855, 1.18760409054941, 0.190422579698768, -2.79013189474126, 1.78440369318915, 1.76759766962424, -0.382078014138221, 0.337068442905171, 1.42244200085645, -0.382337659274782, 1.77717225422036, 1.5667240658887, -0.0648724566595706, 1.40835348803613, -1.07517031384426, 2.0448771635733, 0.20387928210247, -1.35210249661022, -0.586028724744774, -0.84201349457584, 1.90258131506353, 0.411606784318087, -0.439901415814671, 0.400413556728919, -0.522642332863673, 0.488397482651255, 1.51572329544617, 1.52262212337139, -0.205771811432765, -0.358852846817045, 2.97812490826596, 2.27471004314199, -1.73123416913395, -1.01084567421892, 1.110914474459, -0.56793074637455, 1.0457947568757, 0.189080638316264, 2.42007503599557, 0.591808809392759, 1.69135034688281, -1.40091447368174, 0.443539056450542, -1.29350918787999, 1.37109855555011, 0.910485433286087, 0.393159902321618, 0.52588872717339, -0.0699614188535208, -0.205053614911856, -0.121870541194142, 0.948639553493953, -2.55181124672349, -1.62116974064653, -0.927339136556927, -0.0293829057582144, -1.08857253360696, 0.436893948712147, -1.25929831720187, 1.66427394651085, -0.721911619253202, -0.187086780075112, -0.972907200749254, -2.49479634001194, 0.292516580055634, 0.631803837727804, 2.23814166613728, -1.83270431658743, -0.05769627797699, -0.666103978831666, -1.01907287013042, 0.479112620179818, -1.13589709492181, 2.18379229837681, 1.30267223459711, 3.10565034266606, 1.4545741281931, 1.49958177865965, 0.113309221745668, -1.75553538137332, 0.243298221611488, -0.599070157367475, 2.18711836254605, 2.61108014497317, 1.04904278673023, 2.57766985911455, 0.120403846438697, 2.91873222356864, 1.90846716393393, 1.35400218922492, 3.55370994373775, 3.48346062298992, 0.968735809911961, 2.14174807183156, 1.32808583874811, 2.16875563049545, 0.0929685385249678, 2.22990211084164, 1.47934760472248, -0.3262962186754, 0.757439304925393, 0.526780613595748, -1.390667914841, 1.59312449761787, 3.51618200657663, 2.47565428954465, 1.17747469721277, 0.996548478708077, 3.22839349697434, 1.548697329824, 3.64023772923273, 0.885084747149945, 1.48169501229723, -0.39715445631083, 2.15823078284085, 0.896920843058143, 3.9954179341778, 0.289878649966864, 3.46531030502758, 3.09985226481737, -0.975210984984278, 1.73748432962649, 0.0202998463626913, 2.77636804375209, 3.67621364383077, -2.51891159868698, 1.62921382344766, -0.30025454695664, 1.8676058001887, 0.645579086042666, 2.86346682269322, 0.972051047344538, 1.28776117593775, 3.75782604200966, 0.202780599744159, -0.738157146955598, 2.73954177588088, 3.76679370320148, 2.0531571498699, -0.581571362098442, 3.5528765383084, 1.09848339399543, 2.93359377645159, -0.0933376093283611, 1.29039352914662, 3.10380852101533, 1.23415773944705, 2.05415132958319, 1.86615510766519, 1.18501304815952, -0.0391895303226968], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + }, + "aipw": { + "estimate": 1.29320632882505, + "ess": 119.995175832593, + "weights": [0.988121343500857, 1.00319460592959, 0.999326474631137, 0.995659632023005, 0.998606433108666, 0.995983391028368, 0.997905454026944, 0.993067190848352, 0.987447676543771, 1.00722546202046, 0.996098999745148, 0.995760987841229, 0.992551110480417, 1.00946334325196, 1.00213904078528, 0.99656654815872, 0.997707286846142, 1.01031273245395, 0.992552032760831, 1.00246403828434, 0.995416775257639, 1.00074394014255, 0.993277350509223, 1.00046363134566, 0.999387638777348, 0.995436337298158, 1.01002540383716, 0.997641596157817, 1.00142964949572, 1.00306186221818, 0.987568975485235, 0.998901146783634, 0.994210429294235, 1.00267630862969, 0.999430296647091, 0.994417627787845, 1.0006388237355, 0.997913197441718, 1.00474394702358, 0.998439282442818, 0.99809325945128, 0.996577959626109, 1.00807177937616, 0.997758940856196, 0.996365194057743, 0.992843437496156, 0.992961566287008, 1.01184445051893, 0.991172499125968, 1.00607581691, 0.988090533687459, 1.01404534961072, 1.00099910277687, 0.99727551050779, 0.997430908996597, 0.995323694271336, 1.01079586728789, 1.00383782318859, 0.990728396959983, 0.999165942059289, 0.991442682725426, 1.00480350190228, 0.999686306593479, 0.996424768681225, 1.00526516565322, 0.994278461080079, 1.00117423547719, 0.999522222423479, 1.00388724440051, 0.994125269190137, 0.988914815060044, 1.00873014851802, 1.0008025087164, 0.993333496799252, 1.002362069121, 0.995094506867751, 1.01554227443789, 1.01149662395533, 1.00848620855313, 1.00208384713849, 1.0059243759256, 1.00602450217315, 1.00674544005413, 0.994182090996886, 0.998685827626876, 1.00574936778508, 0.994171531804436, 0.990220250456699, 0.994937460421399, 1.00221250585952, 1.00502295762636, 1.00314061240821, 1.01594165711587, 1.00263250272366, 1.00423439904399, 0.999478219836675, 0.9869427697955, 1.00847994595626, 0.999754343768729, 0.990426930175441, 1.00285881267497, 1.00224635046947, 0.999841222797544, 1.01201466214112, 0.996777185410859, 0.994290009671401, 1.00355719467655, 1.00143411749026, 1.00053710082179, 1.01403368944758, 1.00088018587467, 1.00333167103173, 0.998578495421041, 1.00604281551186, 0.995915049400813, 0.996641880507851, 1.00573881013259, 1.0049267688703, 0.998445151041532, 1.00210874012526, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + "dy": [1.38026515156733, 0.408266791145164, -0.339431469645495, -0.682691992758838, -1.82871929455127, 1.14929021958467, 0.406093652733488, 1.39592916302462, 3.44235885109425, -0.820263245202869, -0.0318323352249712, 1.85077537805184, 0.662440201648147, -0.687203070639989, 1.81011852855793, -0.139481071156265, 2.4168721220432, 0.379022022235292, -0.914287504013753, 0.944043804231505, -0.215693960694542, 1.41072414986002, 1.36325294550578, -1.46123322504876, 0.715429531063086, 0.26309981815957, -0.358751783679667, 1.84249495658836, 1.73947344841319, -1.75329820036687, -0.379579864813726, -3.50097882222108, -1.05188660915389, 0.2536693555151, -0.971711160276842, -1.19521084045457, -2.01434628951932, 3.06082555979665, 0.838004826699251, 2.54024677095855, 1.18760409054941, 0.190422579698768, -2.79013189474126, 1.78440369318915, 1.76759766962424, -0.382078014138221, 0.337068442905171, 1.42244200085645, -0.382337659274782, 1.77717225422036, 1.5667240658887, -0.0648724566595706, 1.40835348803613, -1.07517031384426, 2.0448771635733, 0.20387928210247, -1.35210249661022, -0.586028724744774, -0.84201349457584, 1.90258131506353, 0.411606784318087, -0.439901415814671, 0.400413556728919, -0.522642332863673, 0.488397482651255, 1.51572329544617, 1.52262212337139, -0.205771811432765, -0.358852846817045, 2.97812490826596, 2.27471004314199, -1.73123416913395, -1.01084567421892, 1.110914474459, -0.56793074637455, 1.0457947568757, 0.189080638316264, 2.42007503599557, 0.591808809392759, 1.69135034688281, -1.40091447368174, 0.443539056450542, -1.29350918787999, 1.37109855555011, 0.910485433286087, 0.393159902321618, 0.52588872717339, -0.0699614188535208, -0.205053614911856, -0.121870541194142, 0.948639553493953, -2.55181124672349, -1.62116974064653, -0.927339136556927, -0.0293829057582144, -1.08857253360696, 0.436893948712147, -1.25929831720187, 1.66427394651085, -0.721911619253202, -0.187086780075112, -0.972907200749254, -2.49479634001194, 0.292516580055634, 0.631803837727804, 2.23814166613728, -1.83270431658743, -0.05769627797699, -0.666103978831666, -1.01907287013042, 0.479112620179818, -1.13589709492181, 2.18379229837681, 1.30267223459711, 3.10565034266606, 1.4545741281931, 1.49958177865965, 0.113309221745668, -1.75553538137332, 0.243298221611488, -0.599070157367475, 2.18711836254605, 2.61108014497317, 1.04904278673023, 2.57766985911455, 0.120403846438697, 2.91873222356864, 1.90846716393393, 1.35400218922492, 3.55370994373775, 3.48346062298992, 0.968735809911961, 2.14174807183156, 1.32808583874811, 2.16875563049545, 0.0929685385249678, 2.22990211084164, 1.47934760472248, -0.3262962186754, 0.757439304925393, 0.526780613595748, -1.390667914841, 1.59312449761787, 3.51618200657663, 2.47565428954465, 1.17747469721277, 0.996548478708077, 3.22839349697434, 1.548697329824, 3.64023772923273, 0.885084747149945, 1.48169501229723, -0.39715445631083, 2.15823078284085, 0.896920843058143, 3.9954179341778, 0.289878649966864, 3.46531030502758, 3.09985226481737, -0.975210984984278, 1.73748432962649, 0.0202998463626913, 2.77636804375209, 3.67621364383077, -2.51891159868698, 1.62921382344766, -0.30025454695664, 1.8676058001887, 0.645579086042666, 2.86346682269322, 0.972051047344538, 1.28776117593775, 3.75782604200966, 0.202780599744159, -0.738157146955598, 2.73954177588088, 3.76679370320148, 2.0531571498699, -0.581571362098442, 3.5528765383084, 1.09848339399543, 2.93359377645159, -0.0933376093283611, 1.29039352914662, 3.10380852101533, 1.23415773944705, 2.05415132958319, 1.86615510766519, 1.18501304815952, -0.0391895303226968], + "treatment": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + } + } + } + } +} diff --git a/benchmarks/data/twfeweights_sim_panel.csv b/benchmarks/data/twfeweights_sim_panel.csv new file mode 100644 index 000000000..48fbc4d96 --- /dev/null +++ b/benchmarks/data/twfeweights_sim_panel.csv @@ -0,0 +1,1501 @@ +"unit","period","first_treat","x1","xtv","outcome" +1,1,0,-0.44174194168612,0.962638158757321,0.250725238240098 +1,2,0,-0.44174194168612,0.480873514413711,0.302633430899334 +1,3,0,-0.44174194168612,0.114120072263393,-0.099400179401941 +1,4,0,-0.44174194168612,0.448153732387612,1.47267278381102 +1,5,0,-0.44174194168612,0.0450381423543704,-0.304624682447771 +2,1,0,1.00712662877707,0.809674316987755,1.40448682517955 +2,2,0,1.00712662877707,0.875226098004157,1.11498919693233 +2,3,0,1.00712662877707,1.73308918012076,3.30043194865778 +2,4,0,1.00712662877707,1.94159375407109,3.73624251678568 +2,5,0,1.00712662877707,1.49919266041645,5.00490117694144 +3,1,0,0.179611286037529,0.143495801148179,-0.619592378770517 +3,2,0,0.179611286037529,-0.183975411244885,1.59825802459073 +3,3,0,0.179611286037529,1.30125701337682,2.00613564154948 +3,4,0,0.179611286037529,2.14545926587906,3.02931552915487 +3,5,0,0.179611286037529,1.270347859372,0.626648784956356 +4,1,0,1.13465562548833,0.98082184944363,0.789054631222788 +4,2,0,1.13465562548833,0.757860539178009,0.885976737984897 +4,3,0,1.13465562548833,0.644275492814336,2.72497858093159 +4,4,0,1.13465562548833,0.804994143245251,2.7470035085833 +4,5,0,1.13465562548833,3.1402591500037,5.6594454100412 +5,1,0,-2.56473120432832,-1.42045200997001,-2.47773503860188 +5,2,0,-2.56473120432832,-0.99158787828575,-1.91595275185634 +5,3,0,-2.56473120432832,-0.956287354318694,-1.57386277107317 +5,4,0,-2.56473120432832,-0.173688498361684,-2.32481328982363 +5,5,0,-2.56473120432832,-0.220122926796195,-4.77426084425192 +6,1,0,2.03726526019619,0.906852984645204,3.09145390139369 +6,2,0,2.03726526019619,1.10736421765238,1.28301946741103 +6,3,0,2.03726526019619,1.94654212178223,6.4558523593605 +6,4,0,2.03726526019619,2.29731461571635,3.98797124370253 +6,5,0,2.03726526019619,1.90892019581178,8.63257346295759 +7,1,0,0.982893295690819,0.925972608667814,2.56477125464081 +7,2,0,0.982893295690819,1.72746462392985,1.37542121177932 +7,3,0,0.982893295690819,2.01554559420596,3.15083265246945 +7,4,0,0.982893295690819,1.17634719987377,4.35800530188081 +7,5,0,0.982893295690819,0.760671887747645,4.47518570225275 +8,1,0,1.80195765072669,1.92370507752248,2.46720402294067 +8,2,0,1.80195765072669,1.32616500548667,2.7227903293348 +8,3,0,1.80195765072669,1.28608793752108,2.90070675615349 +8,4,0,1.80195765072669,1.81009744339314,5.80136124967764 +8,5,0,1.80195765072669,1.15760990674598,6.55560198907432 +9,1,0,0.225424255156626,0.763669731908457,1.54288624086341 +9,2,0,0.225424255156626,0.874926359148595,0.836845360548911 +9,3,0,0.225424255156626,1.0117962358522,1.82503853096451 +9,4,0,0.225424255156626,0.603697174472571,-0.416276008333528 +9,5,0,0.225424255156626,1.18445658923149,2.87016682298805 +10,1,0,0.0609610990580548,-0.529661334550992,1.66925745533143 +10,2,0,0.0609610990580548,0.0914889871378843,4.08422527417878 +10,3,0,0.0609610990580548,1.48655808874342,2.35118727045354 +10,4,0,0.0609610990580548,0.00203530191935619,2.33692220382611 +10,5,0,0.0609610990580548,1.28112960064862,3.21562126389664 +11,1,0,0.211622353786371,0.677732585269737,1.01402669670136 +11,2,0,0.211622353786371,-0.455890381301695,1.09790190289759 +11,3,0,0.211622353786371,0.221008710144804,0.642633318493822 +11,4,0,0.211622353786371,0.855358439590161,1.02060294398976 +11,5,0,0.211622353786371,1.46959772659548,2.37526538283031 +12,1,0,-0.106312592206072,-9.23288872419215e-05,-0.658080268881745 +12,2,0,-0.106312592206072,-0.0970387920280995,0.736568103159182 +12,3,0,-0.106312592206072,1.28722421860427,1.36381391065831 +12,4,0,-0.106312592206072,0.493486811724755,0.441330823328568 +12,5,0,-0.106312592206072,0.954292388879426,0.728253826747307 +13,1,0,-0.775220530410582,-0.0485173410173907,0.428116160890274 +13,2,0,-0.775220530410582,0.10663036250203,-0.120754966975773 +13,3,0,-0.775220530410582,-0.0932015246981229,0.830881466674987 +13,4,0,-0.775220530410582,0.645640369635815,2.78067278573257 +13,5,0,-0.775220530410582,0.950987026453864,1.73479528567685 +14,1,0,2.04614622265237,0.509538313746868,-0.195487168960969 +14,2,0,2.04614622265237,1.26460645944908,2.49104556632605 +14,3,0,2.04614622265237,1.61979242853853,3.22054459345334 +14,4,0,2.04614622265237,1.96446721321234,2.94031366637701 +14,5,0,2.04614622265237,1.6819638784622,6.3045203371182 +15,1,0,1.91047856851638,1.41809400206497,0.352933179214503 +15,2,0,1.91047856851638,0.868574620082492,2.14983395101162 +15,3,0,1.91047856851638,1.38087806774412,1.4332857406434 +15,4,0,1.91047856851638,2.7376571119788,3.36504006381984 +15,5,0,1.91047856851638,2.50140408640708,5.0339321437845 +16,1,0,0.0904267516471942,-0.127925449618051,-0.629070962146693 +16,2,0,0.0904267516471942,-0.222640384632004,-0.60311595170131 +16,3,0,0.0904267516471942,0.209835589276016,0.386063603258975 +16,4,0,0.0904267516471942,0.518050979618486,1.34554957607334 +16,5,0,0.0904267516471942,1.35263633341545,1.76253705516832 +17,1,0,-0.978997126733862,-0.430729084042156,1.11006747787362 +17,2,0,-0.978997126733862,0.330410967028457,0.806951193076084 +17,3,0,-0.978997126733862,0.684625215247789,1.422241220599 +17,4,0,-0.978997126733862,0.436699992122361,1.05823352541785 +17,5,0,-0.978997126733862,-0.0965033949600655,-0.28861652804708 +18,1,0,-0.305160444674541,-0.916075051557808,-0.542992489238567 +18,2,0,-0.305160444674541,0.248257475817807,-0.860451526400478 +18,3,0,-0.305160444674541,0.798314680654779,-1.7886253887699 +18,4,0,-0.305160444674541,0.521228762410946,0.221495447624939 +18,5,0,-0.305160444674541,0.603880966224836,0.268782385230717 +19,1,0,-1.29347091631414,-0.440720995998017,-0.0917243454597562 +19,2,0,-1.29347091631414,-0.292457346868644,1.69390714797117 +19,3,0,-1.29347091631414,-0.076852304649137,-1.4267554614469 +19,4,0,-1.29347091631414,0.523168449121407,0.836559020778829 +19,5,0,-1.29347091631414,-0.126845315020416,-0.748902007493232 +20,1,0,-0.86946128396502,-0.731640889278664,0.897492140407103 +20,2,0,-0.86946128396502,-0.634157933584901,-0.356647943745405 +20,3,0,-0.86946128396502,0.414634427219905,-0.102122625271403 +20,4,0,-0.86946128396502,0.735617051894694,-0.493531099100508 +20,5,0,-0.86946128396502,0.0783122168879679,2.04744316684707 +21,1,0,0.193364867469568,0.527132325030444,2.83509885985534 +21,2,0,0.193364867469568,0.129936563195705,0.377414995366493 +21,3,0,0.193364867469568,0.613829784876638,2.64691385949551 +21,4,0,0.193364867469568,-0.120119586049108,2.99668502488452 +21,5,0,0.193364867469568,0.508258887874505,4.53816843932383 +22,1,0,-0.285006996929422,-0.544273797377208,0.61950479499134 +22,2,0,-0.285006996929422,-0.0470278141175047,3.0416910941977 +22,3,0,-0.285006996929422,0.307232669634666,3.87159719910662 +22,4,0,-0.285006996929422,1.01711272566063,2.77876159208761 +22,5,0,-0.285006996929422,1.31564072955683,3.62013111033041 +23,1,0,-0.200733998184601,0.406863652048468,0.0824709662570302 +23,2,0,-0.200733998184601,-0.0257062143537835,0.394833336515454 +23,3,0,-0.200733998184601,0.592886053520876,1.23455541173059 +23,4,0,-0.200733998184601,0.110099171857438,2.54019055351788 +23,5,0,-0.200733998184601,0.963691513380404,2.35389561030956 +24,1,0,1.24328569249421,0.92191360816656,1.14586532408122 +24,2,0,1.24328569249421,1.47628244052515,1.97289272773888 +24,3,0,1.24328569249421,0.727658826710546,2.37572049221841 +24,4,0,1.24328569249421,1.32540192428514,6.1798378307657 +24,5,0,1.24328569249421,1.99608430464066,7.76758515003412 +25,1,0,0.472298422553046,0.451130953214354,2.3849436399372 +25,2,0,0.472298422553046,0.393597463425591,-0.0898224276117208 +25,3,0,0.472298422553046,1.25037077852183,2.52589921950728 +25,4,0,0.472298422553046,0.523272852971633,2.37917100316457 +25,5,0,0.472298422553046,0.962139498902349,4.36517881139554 +26,1,0,0.416224904614732,-0.101584055903843,-0.579744769633871 +26,2,0,0.416224904614732,0.400325561772782,-0.972234260476899 +26,3,0,0.416224904614732,0.138863379638057,1.59661513491138 +26,4,0,0.416224904614732,0.397912352674014,0.742603700730964 +26,5,0,0.416224904614732,0.920278133420186,0.713813878237303 +27,1,0,-1.20810463188673,-0.656731791542808,0.112586405323568 +27,2,0,-1.20810463188673,-0.103560125599836,1.12355202824385 +27,3,0,-1.20810463188673,-0.197567905296184,1.53340035807994 +27,4,0,-1.20810463188673,0.974601656086776,-0.432564528542204 +27,5,0,-1.20810463188673,-0.0897357035583219,-0.828997632235555 +28,1,0,-1.43723667380504,0.0565108804189347,-1.84738581883577 +28,2,0,-1.43723667380504,-0.0297675553081633,-0.894498560080979 +28,3,0,-1.43723667380504,-0.0790701797677419,0.0220988662434747 +28,4,0,-1.43723667380504,-0.224757725463843,-0.847105774844605 +28,5,0,-1.43723667380504,0.166110286931856,-2.68630843144862 +29,1,0,1.08387441716567,0.401051829495605,2.54055593664914 +29,2,0,1.08387441716567,0.129584432966412,-0.290371476862269 +29,3,0,1.08387441716567,0.803794134135084,3.27778845957416 +29,4,0,1.08387441716567,0.930893075272324,4.61666902997821 +29,5,0,1.08387441716567,1.66326112137049,4.37994629191061 +30,1,0,1.14107632772678,-0.130216125775587,1.42153467508241 +30,2,0,1.14107632772678,0.543543313638396,1.82708108795342 +30,3,0,1.14107632772678,0.761496767847219,5.94565127190796 +30,4,0,1.14107632772678,1.11210754956847,3.8284928975795 +30,5,0,1.14107632772678,2.12456034096161,4.35152164443134 +31,1,0,-1.68363239672603,-0.394676649711449,1.23436313536159 +31,2,0,-1.68363239672603,-0.476159350468402,-1.2717016524923 +31,3,0,-1.68363239672603,-0.066870496453248,-1.85565601122068 +31,4,0,-1.68363239672603,0.69531782960049,0.800860195139755 +31,5,0,-1.68363239672603,0.252916979891308,0.5339463200228 +32,1,0,0.35862773442386,-0.254987375618001,-0.0878826690809986 +32,2,0,0.35862773442386,0.402486642358358,1.00755257360309 +32,3,0,0.35862773442386,0.973937634716594,1.63709582600101 +32,4,0,0.35862773442386,1.73454911285787,-1.18345432941171 +32,5,0,0.35862773442386,1.95248503861603,3.25719711630834 +33,1,0,1.03062394978694,1.1300648931605,0.251863182842662 +33,2,0,1.03062394978694,0.764447104885131,2.35818680820894 +33,3,0,1.03062394978694,0.81496241040219,-0.240462101427006 +33,4,0,1.03062394978694,1.23953989524317,2.39602149767805 +33,5,0,1.03062394978694,0.868796603908667,4.07416204116012 +34,1,0,-1.36021422621808,-0.037663969198275,0.281343353835558 +34,2,0,-1.36021422621808,-1.26017420369277,0.802690839736971 +34,3,0,-1.36021422621808,0.246925333860566,-0.703061697563873 +34,4,0,-1.36021422621808,-0.16975234477317,1.32797213567783 +34,5,0,-1.36021422621808,0.00740586453438835,0.44762238971308 +35,1,0,2.07462580052836,1.45675082321133,-0.57933493901855 +35,2,0,2.07462580052836,1.20926972518549,1.55463054174886 +35,3,0,2.07462580052836,2.34845238546939,1.6631742348656 +35,4,0,2.07462580052836,2.52239895186696,2.51391502849078 +35,5,0,2.07462580052836,1.52963756522155,4.43956443092784 +36,1,0,-0.624898820481009,0.322071656054509,-1.51999549840851 +36,2,0,-0.624898820481009,-0.436622498509552,-0.401234013390455 +36,3,0,-0.624898820481009,0.173813685252127,-0.0621678249348833 +36,4,0,-0.624898820481009,0.679651321432316,-0.737748502907965 +36,5,0,-0.624898820481009,0.716835039913072,0.736328659584627 +37,1,0,2.30895941653656,1.9742116653891,-2.10332094044947 +37,2,0,2.30895941653656,1.59260370469019,0.821201875196453 +37,3,0,2.30895941653656,2.61155608501098,1.66217100280885 +37,4,0,2.30895941653656,2.52595052377694,2.47477397356914 +37,5,0,2.30895941653656,2.54572354575135,2.57388225810638 +38,1,0,0.468838681888286,0.343993769445637,2.30521872621672 +38,2,0,0.468838681888286,0.641957713818,2.87394510695166 +38,3,0,0.468838681888286,0.880554440811486,4.3703467296499 +38,4,0,0.468838681888286,0.706193181423138,5.39737846119355 +38,5,0,0.468838681888286,1.30558403199082,5.23379128440331 +39,1,0,-0.427218215500197,0.566084390858194,2.13097968656818 +39,2,0,-0.427218215500197,-0.918399310828481,0.413215180305786 +39,3,0,-0.427218215500197,0.118235771072415,2.57020147610967 +39,4,0,-0.427218215500197,0.340995712937832,2.74807377011147 +39,5,0,-0.427218215500197,0.496211049768828,2.9555621794116 +40,1,0,0.777782496536268,0.300747632479431,-0.368684990628421 +40,2,0,0.777782496536268,1.0010007622223,0.595206586920402 +40,3,0,0.777782496536268,0.47188348651664,1.30591749207004 +40,4,0,0.777782496536268,1.86755346941996,3.31554734968523 +40,5,0,0.777782496536268,0.858892873747311,2.75457375011982 +41,1,0,-0.989113034685617,-0.134240789368439,-1.36994283379942 +41,2,0,-0.989113034685617,0.506570270785846,1.0534061137458 +41,3,0,-0.989113034685617,-0.895973224849036,0.27274725887401 +41,4,0,-0.989113034685617,1.31802792818807,0.26632977281089 +41,5,0,-0.989113034685617,0.454944454611298,2.40853446851008 +42,1,0,1.15840793985001,0.124868418660003,0.259801217583987 +42,2,0,1.15840793985001,1.87953027894752,2.17974265446549 +42,3,0,1.15840793985001,0.673567775709603,2.42638252821395 +42,4,0,1.15840793985001,0.63133400895871,2.89643454446268 +42,5,0,1.15840793985001,0.951863725906572,4.47572451802154 +43,1,0,2.67054324703151,2.31680906226424,1.05064037303325 +43,2,0,2.67054324703151,1.96713340733527,4.01260734361365 +43,3,0,2.67054324703151,2.01331188229246,1.02095053233103 +43,4,0,2.67054324703151,2.61485776901755,4.79120104206803 +43,5,0,2.67054324703151,2.06056399005803,7.36148928041125 +44,1,0,0.515574279047854,0.121900480753256,0.12827299244454 +44,2,0,0.515574279047854,0.443740380381531,0.669446613371002 +44,3,0,0.515574279047854,1.41716460656681,2.88575234491894 +44,4,0,0.515574279047854,1.06078400139549,-0.328837067278666 +44,5,0,0.515574279047854,0.846696081449088,1.42436157265577 +45,1,0,0.354081082960897,0.671167680765342,2.13948449234028 +45,2,0,0.354081082960897,0.293960347886397,1.20288799131482 +45,3,0,0.354081082960897,0.433020579196941,2.56751459819996 +45,4,0,0.354081082960897,0.845873361595047,3.54626002487861 +45,5,0,0.354081082960897,0.672652897873653,4.50017565808538 +46,1,0,1.52337503986612,1.32146148533203,3.34298891732902 +46,2,0,1.52337503986612,1.16076099809997,4.24667620804936 +46,3,0,1.52337503986612,1.03398841109356,8.15898555074743 +46,4,0,1.52337503986612,0.994797262587139,8.0779901676523 +46,5,0,1.52337503986612,1.1249651792548,5.68266893304037 +47,1,0,-0.483989426883416,0.574105356443331,-3.20613712871425 +47,2,0,-0.483989426883416,0.208918900884307,0.00649019809915752 +47,3,0,-0.483989426883416,0.741012271351296,0.757311787415 +47,4,0,-0.483989426883416,0.288505807018783,-0.62067301933059 +47,5,0,-0.483989426883416,0.278450260206523,0.549664430944102 +48,1,0,-0.241224470517284,0.538518235254474,0.562488211839541 +48,2,0,-0.241224470517284,0.745322403561392,-0.207881642487272 +48,3,0,-0.241224470517284,0.424900111677857,1.17191392468433 +48,4,0,-0.241224470517284,0.31091296192471,0.890812850085176 +48,5,0,-0.241224470517284,0.603363988372063,0.194681618635391 +49,1,0,-0.502923176484982,-0.845613766548061,-0.0280330536253708 +49,2,0,-0.502923176484982,-0.525656173107788,0.471650194793933 +49,3,0,-0.502923176484982,-0.27163538685412,1.77038444487592 +49,4,0,-0.502923176484982,0.748563048156958,1.06206817243644 +49,5,0,-0.502923176484982,0.862683732520299,1.87497630545043 +50,1,0,0.331707056486104,1.43310608470543,-1.03629025822675 +50,2,0,0.331707056486104,1.47412858157122,0.590271437555205 +50,3,0,0.331707056486104,0.00993825640601731,0.576019539322155 +50,4,0,0.331707056486104,1.18446092385485,1.06859243230635 +50,5,0,0.331707056486104,1.36296624104844,0.823833814241365 +51,1,0,-1.28376827728359,-0.054772706538596,-0.112182614259992 +51,2,0,-1.28376827728359,-0.647495854943373,-0.94712105801857 +51,3,0,-1.28376827728359,0.240043184837449,-0.00441638364042768 +51,4,0,-1.28376827728359,0.0606767301153908,0.875661792221071 +51,5,0,-1.28376827728359,0.379524142127041,1.22049460268803 +52,1,0,0.0268264038406972,0.0813711067876824,0.796399841450881 +52,2,0,0.0268264038406972,0.43253565739872,1.347067415838 +52,3,0,0.0268264038406972,1.09224494984207,1.34411702360282 +52,4,0,0.0268264038406972,0.775188384647136,2.98262612542836 +52,5,0,0.0268264038406972,0.413154429761364,2.67115624971039 +53,1,0,-1.88861627689539,-0.98294737042299,0.975402024308385 +53,2,0,-1.88861627689539,-0.364933260882153,-1.36155509419646 +53,3,0,-1.88861627689539,-1.28010414184404,-0.0953612032279289 +53,4,0,-1.88861627689539,-0.26285694011793,0.645459537255967 +53,5,0,-1.88861627689539,0.320342118934647,-1.34617214380709 +54,1,0,-0.758160785176121,-0.904859165411716,-1.92964585465319 +54,2,0,-0.758160785176121,0.426418257280298,-1.49702195088084 +54,3,0,-0.758160785176121,-0.172965732589018,0.573031919060233 +54,4,0,-0.758160785176121,0.501441761447204,0.60555176976647 +54,5,0,-0.758160785176121,0.748841788556021,0.386565229532384 +55,1,0,0.772657673186417,1.26108129853558,0.848326823659983 +55,2,0,0.772657673186417,0.994286367108801,0.533785646432795 +55,3,0,0.772657673186417,0.132121885819294,1.8393490008326 +55,4,0,0.772657673186417,1.4440198335426,4.31073338025935 +55,5,0,0.772657673186417,1.0421026037683,3.91593091869044 +56,1,0,0.0711352172241156,1.10908756660978,-1.62655633290739 +56,2,0,0.0711352172241156,0.34083565612227,-0.4916460672834 +56,3,0,0.0711352172241156,0.504409497242438,0.466038589769887 +56,4,0,0.0711352172241156,0.204265125308877,-0.613721113111909 +56,5,0,0.0711352172241156,1.43138380470924,0.955380118911139 +57,1,0,-0.794980135199814,-1.0945752615964,-0.419352393006358 +57,2,0,-0.794980135199814,-0.015679604949227,-2.37572615892288 +57,3,0,-0.794980135199814,0.782093277273994,-0.2996905359297 +57,4,0,-0.794980135199814,0.491006375103466,2.1447829543285 +57,5,0,-0.794980135199814,1.00082663497421,-0.477142427035938 +58,1,0,-1.02181782285825,-0.489529600000614,1.87489097842735 +58,2,0,-1.02181782285825,-0.634278175900665,-0.00496069455539061 +58,3,0,-1.02181782285825,0.0812610954568617,3.09374637476232 +58,4,0,-1.02181782285825,1.76062570141884,0.47159533276173 +58,5,0,-1.02181782285825,-0.118990402253873,2.51810462575589 +59,1,0,-0.798010454873102,0.0609546958435624,-2.36229581308972 +59,2,0,-0.798010454873102,0.454243995946293,0.0438796181667978 +59,3,0,-0.798010454873102,0.190314900617629,-1.03078925055765 +59,4,0,-0.798010454873102,0.449234077715798,-0.817729703997447 +59,5,0,-0.798010454873102,0.84368503725751,0.705517798639742 +60,1,0,-0.0306085669312452,0.452616374413926,-0.102673703149613 +60,2,0,-0.0306085669312452,0.813949076299101,-0.434575720306834 +60,3,0,-0.0306085669312452,1.4233380723612,1.29161760939601 +60,4,0,-0.0306085669312452,0.355689837861843,1.56195409402315 +60,5,0,-0.0306085669312452,0.965951055191722,0.909229841469014 +61,1,0,0.067078462093333,-1.09774855868318,0.0382140984881273 +61,2,0,0.067078462093333,-0.0734720467411606,0.173142949330493 +61,3,0,0.067078462093333,-1.19588725717607,0.359042696049106 +61,4,0,0.067078462093333,1.3856053140362,0.427910828219115 +61,5,0,0.067078462093333,1.44251835963329,3.56206647898682 +62,1,0,-1.52731617743477,-1.47707966252442,0.52878696784946 +62,2,0,-1.52731617743477,0.6725293667928,-0.713013683979517 +62,3,0,-1.52731617743477,-0.801863611756564,0.166830610735084 +62,4,0,-1.52731617743477,0.104227361969105,0.458581632509362 +62,5,0,-1.52731617743477,-0.0939253723866919,2.61021076307 +63,1,0,-0.576873399474489,0.568469415283613,-0.0885467512739461 +63,2,0,-0.576873399474489,-0.63462889673578,0.841368442972864 +63,3,0,-0.576873399474489,0.499041270194032,0.730812734245097 +63,4,0,-0.576873399474489,0.46564016095323,0.379790557893517 +63,5,0,-0.576873399474489,0.727842048483113,0.879392177634746 +64,1,0,-0.46221481649607,0.128333789084384,-0.810771268931039 +64,2,0,-0.46221481649607,0.370204400190163,-0.442277924307933 +64,3,0,-0.46221481649607,-0.0845670681314973,0.425221145845222 +64,4,0,-0.46221481649607,0.0289259292010756,1.92641758452772 +64,5,0,-0.46221481649607,0.832360426764675,1.76116093310025 +65,1,0,0.315703776453431,0.185119261555453,2.44859995246444 +65,2,0,0.315703776453431,1.132464076799,2.36264388338644 +65,3,0,0.315703776453431,0.804936552055527,4.33952964926613 +65,4,0,0.315703776453431,1.63738405965253,3.45633927678726 +65,5,0,0.315703776453431,0.654315798187792,2.29243512074624 +66,1,0,1.37658808505767,0.937250197886316,2.81324292738336 +66,2,0,1.37658808505767,1.31236937691446,1.43950882543872 +66,3,0,1.37658808505767,0.817377287895898,3.55004984527523 +66,4,0,1.37658808505767,0.697002957276908,5.44502481471952 +66,5,0,1.37658808505767,1.36246318909377,6.13221143906437 +67,1,0,1.42681697200005,0.929878026545164,1.29641812279912 +67,2,0,1.42681697200005,1.47078818640231,2.49949074805398 +67,3,0,1.42681697200005,1.39010723206942,2.28621088692472 +67,4,0,1.42681697200005,1.4994344602743,3.12588315590834 +67,5,0,1.42681697200005,2.07835117603224,5.93525294343137 +68,1,0,1.35308419041603,0.513814763134145,2.6808262083581 +68,2,0,1.35308419041603,0.71319979591278,3.11853804665789 +68,3,0,1.35308419041603,1.51616580195878,3.70939790832736 +68,4,0,1.35308419041603,0.303962610691678,5.32893930278764 +68,5,0,1.35308419041603,0.61382915360482,4.76798985640524 +69,1,0,1.79859786884555,0.714140771413365,-1.9749036071355 +69,2,0,1.79859786884555,1.75496633936581,1.15033972954931 +69,3,0,1.79859786884555,1.86609357285324,1.43700928442477 +69,4,0,1.79859786884555,1.03796845123607,4.69802297627656 +69,5,0,1.79859786884555,2.80748043992152,4.19758682907294 +70,1,0,-0.333836240206274,-0.271022425491758,1.63473212396469 +70,2,0,-0.333836240206274,1.03771957276743,2.02484454029549 +70,3,0,-0.333836240206274,-0.330129660933743,0.691713233135543 +70,4,0,-0.333836240206274,0.920392687785187,4.37086187320388 +70,5,0,-0.333836240206274,0.555092886752288,2.30949232147289 +71,1,0,-1.63895398153686,-1.18730170194834,2.09459510693227 +71,2,0,-1.63895398153686,0.0659017407261382,0.30490883430922 +71,3,0,-1.63895398153686,-0.62447901592315,0.471122244746123 +71,4,0,-1.63895398153686,-0.915053299758479,-0.865034452672139 +71,5,0,-1.63895398153686,1.14578444726516,0.986950199835889 +72,1,0,0.642543716487307,1.36176968092239,-1.81422208681334 +72,2,0,0.642543716487307,0.748030229144181,1.01388214037359 +72,3,0,0.642543716487307,0.82997265384777,0.0323834944231848 +72,4,0,0.642543716487307,0.880074784091964,1.82932376746001 +72,5,0,0.642543716487307,2.12456495392237,1.75625293742539 +73,1,0,1.8104991643694,0.841870657072021,2.72974425521786 +73,2,0,1.8104991643694,1.45889744701685,3.88174699419143 +73,3,0,1.8104991643694,1.85451418559813,2.80083656747205 +73,4,0,1.8104991643694,1.71168475350052,5.04857246953356 +73,5,0,1.8104991643694,2.09495458053712,6.84980828621333 +74,1,0,-0.932126399374389,-0.568022636104557,2.76205407969921 +74,2,0,-0.932126399374389,0.154151225212912,2.56941894156864 +74,3,0,-0.932126399374389,-0.45323497987067,0.131049516259385 +74,4,0,-0.932126399374389,0.707366959033985,2.23782040811992 +74,5,0,-0.932126399374389,1.2179979556303,3.53842169670308 +75,1,0,0.666231891524512,1.08694657400816,1.87974063555088 +75,2,0,0.666231891524512,1.27378252924364,2.53844953486072 +75,3,0,0.666231891524512,1.84808761734133,2.06146755721079 +75,4,0,0.666231891524512,1.2250029565835,4.65112423855464 +75,5,0,0.666231891524512,2.16385717182326,5.81266126562459 +76,1,0,0.341678631484337,1.06347467961005,0.883641144348533 +76,2,0,0.341678631484337,0.424007246284149,0.927200899437488 +76,3,0,0.341678631484337,0.463634111552505,1.09297461415004 +76,4,0,0.341678631484337,0.883946377784091,1.35138421699445 +76,5,0,0.341678631484337,0.846805066898023,4.35549048057971 +77,1,0,0.637620706206215,1.13509768737945,1.43068721208062 +77,2,0,0.637620706206215,0.119022913744544,4.02884996915181 +77,3,0,0.637620706206215,1.78409687300482,5.37165267440474 +77,4,0,0.637620706206215,0.572587375028069,4.54884339641776 +77,5,0,0.637620706206215,1.30157744988052,4.66081396519702 +78,1,0,0.916740129944944,0.527388555742362,1.67885626210288 +78,2,0,0.916740129944944,1.6516624227088,2.88404956264771 +78,3,0,0.916740129944944,1.57621531218402,5.17729353626615 +78,4,0,0.916740129944944,1.12169573790953,6.39677493288079 +78,5,0,0.916740129944944,1.56442333890756,3.51684585995395 +79,1,0,0.678393345349387,0.981084168977522,-0.303125990400053 +79,2,0,0.678393345349387,0.00845983067858702,3.77063304832269 +79,3,0,0.678393345349387,0.985461654447826,1.28792599622788 +79,4,0,0.678393345349387,1.37335705414109,3.93901911601086 +79,5,0,0.678393345349387,1.59918286286575,4.20477704101249 +80,1,0,-0.614505923587853,-0.074124901025198,0.557946043511742 +80,2,0,-0.614505923587853,0.0775885147622684,3.20589794298296 +80,3,0,-0.614505923587853,-0.129964058314692,1.43190426473233 +80,4,0,-0.614505923587853,1.12112075034697,3.91574997314387 +80,5,0,-0.614505923587853,0.349774794978119,2.92434398626071 +81,1,0,-0.827776110905095,0.0779920496627954,2.20724033209597 +81,2,0,-0.827776110905095,0.329874381465543,-0.343896532607536 +81,3,0,-0.827776110905095,-0.509582604067608,0.184330306231341 +81,4,0,-0.827776110905095,0.425631993327435,-0.677124424118725 +81,5,0,-0.827776110905095,0.576793045090816,5.25417091189951 +82,1,0,0.861142171302151,0.534905942272975,-0.559829415854437 +82,2,0,0.861142171302151,0.421001725699289,-0.640432437832771 +82,3,0,0.861142171302151,1.33334862780767,1.15221010623295 +82,4,0,0.861142171302151,1.51902787603164,1.84609545714379 +82,5,0,0.861142171302151,2.27137340338279,1.20605568714721 +83,1,0,0.616000967810865,1.56647356845175,0.856810537009867 +83,2,0,0.616000967810865,0.418959285740418,3.0293435997056 +83,3,0,0.616000967810865,1.71178089113956,4.1807485972266 +83,4,0,0.616000967810865,1.75291581545166,3.61295843048961 +83,5,0,0.616000967810865,1.95198829511699,5.40410230770897 +84,1,0,-0.364563738233122,0.729912191174741,-1.54643580105865 +84,2,0,-0.364563738233122,0.270810379458042,0.662855640387282 +84,3,0,-0.364563738233122,0.46985768052808,3.08624127314465 +84,4,0,-0.364563738233122,0.471373212800752,0.515624799993429 +84,5,0,-0.364563738233122,0.781470542382374,1.70977184890727 +85,1,0,0.143653481491035,0.940535155473577,-0.0595363250531367 +85,2,0,0.143653481491035,-0.338084292631134,-0.805588718694135 +85,3,0,0.143653481491035,1.27753543499624,0.00832597427848673 +85,4,0,0.143653481491035,0.89192092464685,2.03135738913338 +85,5,0,0.143653481491035,0.703431750258417,0.599027040421859 +86,1,0,-0.131220650539475,-0.0817446182852218,-0.396935575601526 +86,2,0,-0.131220650539475,0.191895364526733,-0.194650738734955 +86,3,0,-0.131220650539475,0.788377198501869,3.73853028280585 +86,4,0,-0.131220650539475,1.02415048291942,1.91396930977068 +86,5,0,-0.131220650539475,1.03878595672618,2.1859552336763 +87,1,0,-0.333066256749381,1.06674969434541,-0.0657657799846833 +87,2,0,-0.333066256749381,0.149647369032651,0.498161299881925 +87,3,0,-0.333066256749381,0.581926896494485,1.13646345708566 +87,4,0,-0.333066256749381,-0.247133860668094,-0.175924284009217 +87,5,0,-0.333066256749381,1.46732523994061,1.91561016686449 +88,1,0,-0.434370693446027,-0.437377299668023,-0.772574889112886 +88,2,0,-0.434370693446027,-0.152739483160924,-0.568834744275269 +88,3,0,-0.434370693446027,0.951913298045205,0.385469624381133 +88,4,0,-0.434370693446027,1.1189528375249,-0.057927566466312 +88,5,0,-0.434370693446027,0.568042594475733,0.443220743369422 +89,1,0,-0.141474869850029,0.0801484231476475,0.0270269788568515 +89,2,0,-0.141474869850029,0.0481236129066036,0.205532603073162 +89,3,0,-0.141474869850029,0.995260603221087,0.651653657935192 +89,4,0,-0.141474869850029,1.32639060956188,1.59469437463446 +89,5,0,-0.141474869850029,0.645906816771665,2.15442315953645 +90,1,0,1.03227036316416,0.63763451526355,-1.25422979344628 +90,2,0,1.03227036316416,0.405889210017968,-0.110219289419371 +90,3,0,1.03227036316416,0.458014774582056,0.982330971708697 +90,4,0,1.03227036316416,0.852538382878488,3.76935892335802 +90,5,0,1.03227036316416,0.962265711370071,2.56238146466164 +91,1,0,-0.486491594070801,-0.172045064925041,1.83038665225263 +91,2,0,-0.486491594070801,0.384618723183606,0.0474591275790941 +91,3,0,-0.486491594070801,0.531878509700311,1.7092334560639 +91,4,0,-0.486491594070801,0.793638805747236,2.5679967220646 +91,5,0,-0.486491594070801,0.722297893770697,3.12470764883272 +92,1,0,0.219980494850729,0.0734058917248482,0.384234866874619 +92,2,0,0.219980494850729,-0.219734183232657,0.903091028630454 +92,3,0,0.219980494850729,0.682596952495566,2.12751213821996 +92,4,0,0.219980494850729,1.23869541320448,3.5612634595244 +92,5,0,0.219980494850729,0.578532061197268,3.27222425245696 +93,1,0,0.288608812761447,0.421231189913603,-1.40212126529636 +93,2,0,0.288608812761447,0.454485182510659,0.422698666792128 +93,3,0,0.288608812761447,1.31865477532447,0.115353686002623 +93,4,0,0.288608812761447,1.08220992063066,2.67936764198457 +93,5,0,0.288608812761447,0.959458903140813,2.13821853667061 +94,1,0,-1.61902954772177,-1.20480995091084,1.82409496936011 +94,2,0,-1.61902954772177,-0.930824167709811,1.42101239978627 +94,3,0,-1.61902954772177,-0.584839870198029,1.78332374092068 +94,4,0,-1.61902954772177,-0.346404263307608,-0.227553617442285 +94,5,0,-1.61902954772177,1.1729349153929,1.9498809084647 +95,1,0,0.222271128189547,-0.0387473417629938,0.858725090646077 +95,2,0,0.222271128189547,0.767954290471442,2.69923913505218 +95,3,0,0.222271128189547,0.605075552915976,1.99193702670537 +95,4,0,0.222271128189547,1.3558238746813,2.73108721331929 +95,5,0,0.222271128189547,1.63139262575401,0.783913778717177 +96,1,0,1.54600822113029,2.02001061311009,2.62936331979604 +96,2,0,1.54600822113029,1.35702096802835,1.96438966667089 +96,3,0,1.54600822113029,1.36437355018916,1.82028484401593 +96,4,0,1.54600822113029,1.54806112741881,2.22526558385425 +96,5,0,1.54600822113029,2.88653595933768,5.29452516245512 +97,1,0,0.133697246677124,-0.0718599103232635,1.93759954133085 +97,2,0,0.133697246677124,0.243856103638007,1.83522434020185 +97,3,0,0.133697246677124,0.138526367764456,4.6449042058556 +97,4,0,0.133697246677124,1.8769167656904,3.54647214833668 +97,5,0,0.133697246677124,0.962401521139283,2.9247955599588 +98,1,0,0.513461762623951,0.550111450173374,1.13638082120044 +98,2,0,0.513461762623951,-0.116902446416303,2.0047933773136 +98,3,0,0.513461762623951,1.39013026554457,2.89151767905503 +98,4,0,0.513461762623951,1.5481840147936,2.39962626326609 +98,5,0,0.513461762623951,0.988855636857315,2.77565780113198 +99,1,0,0.0284391113388029,0.689085116598258,0.254359114140023 +99,2,0,0.0284391113388029,0.349968288749272,0.878606062697145 +99,3,0,0.0284391113388029,1.81169809337691,0.440914916417645 +99,4,0,0.0284391113388029,0.983652616962293,2.17316792107343 +99,5,0,0.0284391113388029,1.20691015927357,3.06483425373133 +100,1,0,-1.7202950012414,-0.115879309040799,-1.39743104846185 +100,2,0,-1.7202950012414,0.0489770680107625,-1.04291605039209 +100,3,0,-1.7202950012414,-1.17871267878631,-0.81502457454413 +100,4,0,-1.7202950012414,-0.105520702288695,0.334070265764476 +100,5,0,-1.7202950012414,-0.467384608422146,-0.890554303485205 +101,1,3,-0.657717766276882,-0.94206308645325,0.232488558205459 +101,2,3,-0.657717766276882,0.0705885988157602,0.953684678547186 +101,3,3,-0.657717766276882,0.0953022991421917,4.99112586140753 +101,4,3,-0.657717766276882,0.271992657738819,3.73290731784579 +101,5,3,-0.657717766276882,0.170440318698168,5.85181562906916 +102,1,3,0.606628948683484,-0.13131505600894,2.91452541834592 +102,2,3,0.606628948683484,1.4022497729769,0.261042740854546 +102,3,3,0.606628948683484,0.993231830563238,2.71853186863986 +102,4,3,0.606628948683484,0.884332783542463,4.56588455342279 +102,5,3,0.606628948683484,1.56713473862668,7.09247867741218 +103,1,3,-0.332641963407837,0.154534274202655,0.602279850487755 +103,2,3,-0.332641963407837,0.072432790726706,-0.309540706057066 +103,3,3,-0.332641963407837,1.16818496470408,-0.165286394085044 +103,4,3,-0.332641963407837,0.702511034762439,1.34374356615043 +103,5,3,-0.332641963407837,1.01447715390258,5.3160097148392 +104,1,3,0.469706705466534,0.279153249372299,-0.383474155857604 +104,2,3,0.469706705466534,1.26056420037322,0.450636566145736 +104,3,3,0.469706705466534,0.840143863086957,3.06714331395692 +104,4,3,0.469706705466534,1.5062897407947,5.65655038983017 +104,5,3,0.469706705466534,1.1964368519802,5.11830997961023 +105,1,3,1.66748637673741,1.12973770628659,-1.99633809999194 +105,2,3,1.66748637673741,1.71587553353966,-1.18150760955215 +105,3,3,1.66748637673741,1.51301509979169,3.13175716039968 +105,4,3,1.66748637673741,1.52614573553539,5.51260657016865 +105,5,3,1.66748637673741,1.80360251228863,6.86533385915753 +106,1,3,1.27268901841241,0.410841768659794,0.678968405380034 +106,2,3,1.27268901841241,2.42608384357659,0.996016318495919 +106,3,3,1.27268901841241,1.75308326994184,2.48578861598163 +106,4,3,1.27268901841241,1.50527027515069,6.60521083807514 +106,5,3,1.27268901841241,0.851577609230435,9.15879471695537 +107,1,3,-0.113521083630058,-0.617931847269183,0.389795760489174 +107,2,3,-0.113521083630058,0.00578001996492189,1.24529091081873 +107,3,3,-0.113521083630058,0.522769231917015,3.59142446549231 +107,4,3,-0.113521083630058,0.55896664892714,4.08609378496072 +107,5,3,-0.113521083630058,0.624181425631188,6.65868685406829 +108,1,3,1.42915725643045,1.03946831567814,-1.49277112713376 +108,2,3,1.42915725643045,0.355936899218436,-0.993582726915977 +108,3,3,1.42915725643045,1.3102549176241,0.706937887500567 +108,4,3,1.42915725643045,1.91255925354156,2.4057349385667 +108,5,3,1.42915725643045,1.11706892387658,5.42943529075469 +109,1,3,-0.559679157783233,0.240146108567419,1.85721232954093 +109,2,3,-0.559679157783233,-1.18164851474772,0.598496577056571 +109,3,3,-0.559679157783233,-0.567714929049523,3.8190582793659 +109,4,3,-0.559679157783233,1.58499831755575,3.61974345956697 +109,5,3,-0.559679157783233,0.281509011231452,5.64271736926589 +110,1,3,0.187728838960483,-0.0990282930604425,0.636491412067755 +110,2,3,0.187728838960483,0.522934521191726,-1.27512878217836 +110,3,3,0.187728838960483,0.451864219784705,0.505624681678577 +110,4,3,0.187728838960483,0.778209878446357,2.64560002072363 +110,5,3,0.187728838960483,1.85942863847523,5.79668804951045 +111,1,3,1.68213095275821,0.251934065681341,3.91605951374998 +111,2,3,1.68213095275821,1.35011115331486,3.79188977773365 +111,3,3,1.68213095275821,1.97032026344658,3.08046167492404 +111,4,3,1.68213095275821,2.2589275661647,8.27001943564112 +111,5,3,1.68213095275821,1.81228175371279,7.85436781063261 +112,1,3,-0.0788326245762765,-0.0886876014313262,0.741167712744016 +112,2,3,-0.0788326245762765,0.057345760642664,-0.289974470099434 +112,3,3,-0.0788326245762765,0.217463034910929,1.05732757506079 +112,4,3,-0.0788326245762765,1.61994832884064,5.16748675022915 +112,5,3,-0.0788326245762765,1.10512689283478,6.32704551373198 +113,1,3,-1.11820261456398,-0.326553621508285,4.59619133661404 +113,2,3,-1.11820261456398,0.138424198638174,3.08762952083826 +113,3,3,-1.11820261456398,0.580218918516405,4.11792205698977 +113,4,3,-1.11820261456398,0.180859352511177,5.12177393132146 +113,5,3,-1.11820261456398,0.466131001972245,6.08972404155704 +114,1,3,0.00617399378076931,0.655178842061438,-2.62808216419414 +114,2,3,0.00617399378076931,1.54562520211198,-0.333313638255983 +114,3,3,0.00617399378076931,-0.0719855219840838,0.0862359331011819 +114,4,3,0.00617399378076931,0.444363543616229,2.84552412987694 +114,5,3,0.00617399378076931,0.531577221298498,3.34103605859625 +115,1,3,-0.466490512919715,-0.70628149463559,-1.45817530327387 +115,2,3,-0.466490512919715,0.729198541085584,-1.93903353127595 +115,3,3,-0.466490512919715,0.543408386695873,-0.351268598390575 +115,4,3,-0.466490512919715,1.02986330956025,1.1754453578064 +115,5,3,-0.466490512919715,0.412796845409179,3.51229890894027 +116,1,3,-2.26408332836368,-1.30773888985853,-2.10077614733638 +116,2,3,-2.26408332836368,-1.37763784721948,0.236161311201754 +116,3,3,-2.26408332836368,-1.7425007121992,1.23135889576542 +116,4,3,-2.26408332836368,-0.708621021967853,1.87035942321069 +116,5,3,-2.26408332836368,-1.03635888476242,-0.645611331548352 +117,1,3,0.552125571623184,1.32365285117336,0.779376482838142 +117,2,3,0.552125571623184,0.930668266663576,3.04427235955234 +117,3,3,0.552125571623184,0.622006301346775,3.68654705941641 +117,4,3,0.552125571623184,1.77537177199262,5.84936566524921 +117,5,3,0.552125571623184,1.4463562521855,7.26081776390063 +118,1,3,-0.294573500984481,-0.226854558548949,0.624381599018543 +118,2,3,-0.294573500984481,0.451975050319793,-0.50960478842452 +118,3,3,-0.294573500984481,1.10372122526393,1.56675527682092 +118,4,3,-0.294573500984481,0.820290560391889,3.70060353900024 +118,5,3,-0.294573500984481,0.0496382963428972,4.85596894365002 +119,1,3,-0.723593527799914,-0.505152373633315,2.0676374826663 +119,2,3,-0.723593527799914,0.101750802662789,1.03136110282366 +119,3,3,-0.723593527799914,0.730559754677908,3.41623122468754 +119,4,3,-0.723593527799914,0.00221535064169642,3.78216320855324 +119,5,3,-0.723593527799914,0.636998000740745,4.93833031585068 +120,1,3,0.858310574796443,1.08316871624732,5.23943573915105 +120,2,3,0.858310574796443,0.863887874687908,5.35534446922412 +120,3,3,0.858310574796443,2.11534837934243,6.58543264119611 +120,4,3,0.858310574796443,1.88024199878906,9.8528988127391 +120,5,3,0.858310574796443,1.27582322704727,11.2611305249796 +121,1,3,0.220462861432781,-0.309440648082669,-0.755306621580593 +121,2,3,0.220462861432781,0.627055418074356,-0.023684948685408 +121,3,3,0.220462861432781,0.545149725436819,1.36183809596365 +121,4,3,0.220462861432781,0.792007240646273,3.18699848795418 +121,5,3,0.220462861432781,1.69726445585479,3.95436649310404 +122,1,3,-0.140752859229319,0.177062299482488,-1.76683361804301 +122,2,3,-0.140752859229319,0.304636966807162,0.993079818624433 +122,3,3,-0.140752859229319,0.421561519180845,0.332450286328496 +122,4,3,-0.140752859229319,1.36949693276948,3.85954344361671 +122,5,3,-0.140752859229319,0.568440407673914,3.85447057933448 +123,1,3,1.08154108301988,0.535266128351368,0.779969396251111 +123,2,3,1.08154108301988,0.880289324345213,1.59969760053008 +123,3,3,1.08154108301988,0.756275612370415,4.49990130269742 +123,4,3,1.08154108301988,1.05475695661148,5.66132166010569 +123,5,3,1.08154108301988,0.991482360834123,7.76732143736291 +124,1,3,0.000773525396488833,0.462844594613568,0.841288741599271 +124,2,3,0.000773525396488833,0.602602082712852,0.312361825429711 +124,3,3,0.000773525396488833,0.339182135812421,2.46117172121027 +124,4,3,0.000773525396488833,0.560962815672359,3.19918261277187 +124,5,3,0.000773525396488833,0.445130224054067,6.56037285845182 +125,1,3,-0.212450189809644,0.948086214476399,3.10996825066564 +125,2,3,-0.212450189809644,0.318713140332635,3.16644442473342 +125,3,3,-0.212450189809644,0.680601781620023,3.20618608330843 +125,4,3,-0.212450189809644,1.62789839447148,4.76744325623378 +125,5,3,-0.212450189809644,1.3134135459942,5.12920413147696 +126,1,3,-0.0749218641308017,-0.250989637603912,-0.34160343652017 +126,2,3,-0.0749218641308017,-0.107579672268752,0.194139114567561 +126,3,3,-0.0749218641308017,0.93589473033804,3.58262564440672 +126,4,3,-0.0749218641308017,0.683688042722152,1.72966565599836 +126,5,3,-0.0749218641308017,1.71491065530947,4.19726792441863 +127,1,3,0.627289566110117,1.05729793945288,1.62128596144571 +127,2,3,0.627289566110117,0.973758617212842,2.34129685541675 +127,3,3,0.627289566110117,0.828958721907215,3.88047996241525 +127,4,3,0.627289566110117,1.54879032336665,5.42297645981372 +127,5,3,0.627289566110117,1.48328874793323,8.63035389347065 +128,1,3,0.410545949407018,0.625141759106629,1.94363260911011 +128,2,3,0.410545949407018,0.840869756849677,2.63277909520217 +128,3,3,0.410545949407018,1.69458209211863,6.08657628694377 +128,4,3,0.410545949407018,0.638430284893267,6.45369861408724 +128,5,3,0.410545949407018,2.5733341846153,7.72778285671548 +129,1,3,1.33028430702108,2.13073984612338,1.69376984087629 +129,2,3,1.33028430702108,0.646345800359394,3.21444010789856 +129,3,3,1.33028430702108,2.04773265337806,3.61285827791554 +129,4,3,1.33028430702108,1.67761633423117,5.69764116082381 +129,5,3,1.33028430702108,0.912577561277432,9.44538478230292 +130,1,3,-1.30395806956874,0.140532026865759,-0.905148818564164 +130,2,3,-1.30395806956874,0.1334871419863,0.394705286548389 +130,3,3,-1.30395806956874,0.0860359670190863,3.32062284302592 +130,4,3,-1.30395806956874,0.30340861859709,3.28255261756849 +130,5,3,-1.30395806956874,0.658912821802418,2.56331948055225 +131,1,3,-0.357820487255056,0.604743403188498,-0.951886500117099 +131,2,3,-0.357820487255056,0.647902518744151,-0.167819315846137 +131,3,3,-0.357820487255056,1.10059087835971,0.101181804228307 +131,4,3,-0.357820487255056,0.308431637907341,3.75349531284482 +131,5,3,-0.357820487255056,0.718013535203741,2.89114217148319 +132,1,3,0.48766605796259,0.0140713891864764,-2.21162281394707 +132,2,3,0.48766605796259,0.156165054524314,-0.679269124727664 +132,3,3,0.48766605796259,0.732944740978031,2.81641237256502 +132,4,3,0.48766605796259,1.28017422519403,2.46764691181743 +132,5,3,0.48766605796259,0.938788379132021,4.56142902922779 +133,1,3,0.0965095252581635,-0.424257311714683,-0.906138381171024 +133,2,3,0.0965095252581635,0.682038652664297,0.467549662669626 +133,3,3,0.0965095252581635,0.599960579187045,1.56808656793963 +133,4,3,0.0965095252581635,0.670053117047675,5.99640467483164 +133,5,3,0.0965095252581635,0.00290729768220532,5.61187286084779 +134,1,3,0.470813281504669,-0.0237977219345792,-1.17861719509669 +134,2,3,0.470813281504669,0.0210231463695877,2.26535855456937 +134,3,3,0.470813281504669,1.65875503097192,2.79972362019488 +134,4,3,0.470813281504669,1.4522899241199,6.96850396868235 +134,5,3,0.470813281504669,1.19697537953027,5.53220254058808 +135,1,3,1.67155479620156,1.63003289483879,1.13465670732127 +135,2,3,1.67155479620156,1.66566175054474,2.16928327099793 +135,3,3,1.67155479620156,1.23104108297534,4.62609919171196 +135,4,3,1.67155479620156,2.51547061628803,6.42366307314904 +135,5,3,1.67155479620156,0.773175668804269,6.06302343062228 +136,1,3,-0.0332394453434365,0.0760360749606722,0.0208825919783583 +136,2,3,-0.0332394453434365,0.758090599491503,0.568066216720766 +136,3,3,-0.0332394453434365,0.865406851245269,2.16385713696812 +136,4,3,-0.0332394453434365,0.287846254413097,4.10855697907143 +136,5,3,-0.0332394453434365,0.735172436002937,4.53398714072422 +137,1,3,-0.688225331751058,0.0016693549665899,-0.605796558059025 +137,2,3,-0.688225331751058,0.31564905198807,2.09364296940547 +137,3,3,-0.688225331751058,0.302456958121517,2.91353634798101 +137,4,3,-0.688225331751058,1.4601427357091,3.55511507145684 +137,5,3,-0.688225331751058,0.52377242178836,6.28678207632846 +138,1,3,-0.500122627493091,0.396683358364411,1.06354027076539 +138,2,3,-0.500122627493091,0.147266283149743,-0.416532302563743 +138,3,3,-0.500122627493091,-0.0674183543222071,2.39971802018502 +138,4,3,-0.500122627493091,0.444002742154018,4.72359154487664 +138,5,3,-0.500122627493091,-0.00639513954455206,6.2797345135803 +139,1,3,0.18107731763868,0.358754646027512,2.56815917737501 +139,2,3,0.18107731763868,0.473843315161791,5.09625792076298 +139,3,3,0.18107731763868,0.222826468427173,5.09301261569359 +139,4,3,0.18107731763868,0.998960955021969,5.19457217305691 +139,5,3,0.18107731763868,0.94396796417614,7.6238515902996 +140,1,3,-1.37211800780397,-0.397986200765386,-0.486212418062664 +140,2,3,-1.37211800780397,-1.14777684605006,-0.587349940190411 +140,3,3,-1.37211800780397,0.207813697049092,0.0179085891028778 +140,4,3,-1.37211800780397,-0.0157145368531961,0.718055885197066 +140,5,3,-1.37211800780397,0.141532328048656,3.84922898075712 +141,1,3,-1.19897389691436,0.360461248314332,-1.8152035620548 +141,2,3,-1.19897389691436,-0.542266936350596,-2.95909668834627 +141,3,3,-1.19897389691436,0.99913729191338,0.895701803570273 +141,4,3,-1.19897389691436,0.397516705219247,1.30946775962812 +141,5,3,-1.19897389691436,0.557238702745939,2.26160192220899 +142,1,3,-0.998319924353911,-0.617579580251617,-1.74650801638042 +142,2,3,-0.998319924353911,-0.949218896378564,-0.530887740124697 +142,3,3,-0.998319924353911,0.039191340332905,0.0526537324003162 +142,4,3,-0.998319924353911,1.06166107117482,1.17556748272577 +142,5,3,-0.998319924353911,0.388788578221689,3.55709306215575 +143,1,3,1.20119683314219,0.774166948615702,1.59415308736676 +143,2,3,1.20119683314219,1.60847913451569,1.16604946065365 +143,3,3,1.20119683314219,1.79639042370985,3.97103994157813 +143,4,3,1.20119683314219,0.749734928571424,4.99089995233893 +143,5,3,1.20119683314219,1.4255599566404,9.34890063805875 +144,1,3,1.00628694430582,1.08167301522945,-0.555804553785998 +144,2,3,1.00628694430582,1.28955773107959,-0.794982544165143 +144,3,3,1.00628694430582,0.531935299948853,2.6417560919309 +144,4,3,1.00628694430582,1.96009703859714,3.43572503800884 +144,5,3,1.00628694430582,1.20637425198866,4.34655255306734 +145,1,3,1.67091120380766,1.25583677189642,1.88484533573143 +145,2,3,1.67091120380766,0.697055921239634,0.705532024610885 +145,3,3,1.67091120380766,1.58825290389066,3.24141239581406 +145,4,3,1.67091120380766,1.83616400511528,6.05043141490311 +145,5,3,1.67091120380766,1.28185088085329,5.02523739500611 +146,1,3,-0.379316649007007,-0.0834809234825081,-0.604888670214619 +146,2,3,-0.379316649007007,-0.795085926968904,-0.439042960407943 +146,3,3,-0.379316649007007,0.401301223844121,0.614602642472997 +146,4,3,-0.379316649007007,0.476915753436164,2.40152270032433 +146,5,3,-0.379316649007007,0.130152214243919,3.90321730111349 +147,1,3,0.420754458915134,0.466207147814328,-1.67292060140418 +147,2,3,0.420754458915134,0.569134175687715,-0.828083532691714 +147,3,3,0.420754458915134,0.883067633049473,0.919896016564556 +147,4,3,0.420754458915134,0.531632319632921,3.08518200522015 +147,5,3,0.420754458915134,0.635630282770971,4.59162567793378 +148,1,3,0.0311732664506885,-0.0978878456004359,0.293726126730062 +148,2,3,0.0311732664506885,0.565589374149626,0.569298807450127 +148,3,3,0.0311732664506885,0.778310646490644,1.69791627492246 +148,4,3,0.0311732664506885,0.962303021614927,2.33824377902591 +148,5,3,0.0311732664506885,1.19136094560224,4.05174848811233 +149,1,3,0.512760193053312,0.0915936923166048,0.00633837792554687 +149,2,3,0.512760193053312,0.581983898348367,-0.436480189053884 +149,3,3,0.512760193053312,0.0634121352834593,2.40477462650618 +149,4,3,0.512760193053312,0.569371944296203,4.32343142524583 +149,5,3,0.512760193053312,1.21441640506989,4.73370495644068 +150,1,3,1.87868646637167,0.69125924639008,3.38649600786785 +150,2,3,1.87868646637167,1.31653504546201,3.27059369523601 +150,3,3,1.87868646637167,1.02569583379149,6.39971209529027 +150,4,3,1.87868646637167,1.44661621609673,6.51984488748056 +150,5,3,1.87868646637167,2.02528698187357,9.61899939982145 +151,1,3,-1.84278530939641,-0.644029715762125,-2.22769544227137 +151,2,3,-1.84278530939641,-0.441329046218621,-1.45924645134561 +151,3,3,-1.84278530939641,-0.145124481339132,-1.15662859328522 +151,4,3,-1.84278530939641,-0.168275634333566,-1.17671356983013 +151,5,3,-1.84278530939641,1.1584080631215,-0.828004130089602 +152,1,3,1.72965756391273,0.255517813411765,1.53812815985889 +152,2,3,1.72965756391273,2.03646206475824,3.02640293459076 +152,3,3,1.72965756391273,1.96685280001739,5.0807862101301 +152,4,3,1.72965756391273,1.78778264470661,7.07323562820844 +152,5,3,1.72965756391273,2.63307633311382,9.20377058468654 +153,1,3,-0.29666031962568,-0.286682880554559,-0.905739111548258 +153,2,3,-0.29666031962568,0.0730518169668248,1.83539397771319 +153,3,3,-0.29666031962568,1.05754267911064,2.24979895590196 +153,4,3,-0.29666031962568,0.543774543760072,4.15543805256443 +153,5,3,-0.29666031962568,1.52983625197896,4.26308726987872 +154,1,3,-0.629413689190318,0.727509671424609,3.71401857261263 +154,2,3,-0.629413689190318,1.29760725996956,1.12434826453582 +154,3,3,-0.629413689190318,0.932495976740686,3.01651148831642 +154,4,3,-0.629413689190318,0.864584440262066,3.13699893765196 +154,5,3,-0.629413689190318,0.712000503501498,5.4012578917997 +155,1,3,1.26483805731984,0.101373228065778,0.373539829402359 +155,2,3,1.26483805731984,0.212809068946509,1.58504815689564 +155,3,3,1.26483805731984,0.99358154798726,2.71644229480947 +155,4,3,1.26483805731984,2.25517833481268,4.34058665417517 +155,5,3,1.26483805731984,1.97118903746078,7.74877923012855 +156,1,3,-0.883753614415126,-0.399535191884741,0.0960186166312004 +156,2,3,-0.883753614415126,-0.320901569906869,0.244002366570801 +156,3,3,-0.883753614415126,1.06156750938429,2.24533452927691 +156,4,3,-0.883753614415126,0.461016513134735,1.85726192341166 +156,5,3,-0.883753614415126,0.510702258754348,4.97094755371067 +157,1,3,-0.920994725090723,-0.892479349459743,-0.998800959364927 +157,2,3,-0.920994725090723,-1.15880706003958,0.367783265190597 +157,3,3,-0.920994725090723,-0.34037148362012,1.01156615038342 +157,4,3,-0.920994725090723,0.835538390428648,2.31233387560194 +157,5,3,-0.920994725090723,-0.25552330880026,3.01427008481776 +158,1,3,-0.34893368979522,-0.267555864749471,0.391980485124665 +158,2,3,-0.34893368979522,0.631473827249395,0.118968890864104 +158,3,3,-0.34893368979522,0.401379529995111,2.00487956507939 +158,4,3,-0.34893368979522,0.253640013407772,2.70578554698554 +158,5,3,-0.34893368979522,0.824772396694025,5.63766780336331 +159,1,3,-0.304255532851446,-0.675035468671625,0.975756093026101 +159,2,3,-0.304255532851446,0.0759771386422023,1.40147646679613 +159,3,3,-0.304255532851446,0.230792842617687,1.65368279369012 +159,4,3,-0.304255532851446,0.811272643636225,2.10771822600086 +159,5,3,-0.304255532851446,1.28756246587767,6.54277097690431 +160,1,3,-1.01219093638919,0.127507399755032,-2.35564452433788 +160,2,3,-1.01219093638919,0.268959034351851,-1.51029850831283 +160,3,3,-1.01219093638919,-0.407224246647464,-0.822751398045535 +160,4,3,-1.01219093638919,-0.0518131717045414,2.25240396176116 +160,5,3,-1.01219093638919,0.605259863963078,1.95588826400922 +161,1,3,-0.757717772717969,0.605568531426284,1.07474760513235 +161,2,3,-0.757717772717969,0.0731360455976896,-0.615467370613254 +161,3,3,-0.757717772717969,0.995768606560662,1.33512778948212 +161,4,3,-0.757717772717969,-0.542214426349538,3.73414229763899 +161,5,3,-0.757717772717969,0.376089632978491,4.32273833332243 +162,1,3,-0.633058606184867,0.444474326199027,-0.157864196813987 +162,2,3,-0.633058606184867,-0.358409061943328,1.36588036876563 +162,3,3,-0.633058606184867,0.00717898377558673,1.5208683399469 +162,4,3,-0.633058606184867,0.928888472555571,3.91583595246959 +162,5,3,-0.633058606184867,0.573737524170066,3.11203053101271 +163,1,3,-1.18786324515284,-0.340443408233712,1.14317643363866 +163,2,3,-1.18786324515284,-0.768032906626293,1.95833915002506 +163,3,3,-1.18786324515284,0.272259951460627,0.114323384349211 +163,4,3,-1.18786324515284,-0.298978178385548,0.61384379102693 +163,5,3,-1.18786324515284,0.637413178921166,5.50132822693627 +164,1,3,0.899797869140011,1.05596549859405,-0.21122143226474 +164,2,3,0.899797869140011,2.17406704550966,1.24046961464129 +164,3,3,0.899797869140011,1.44083698995944,2.66189961002648 +164,4,3,0.899797869140011,0.915916560712705,2.36672985820162 +164,5,3,0.899797869140011,1.38588314168802,5.02702750512009 +165,1,3,-0.301129927044137,0.752158340030402,-1.50907794920551 +165,2,3,-0.301129927044137,1.06436019651387,-0.62448442577378 +165,3,3,-0.301129927044137,0.384960393290526,0.832916152884812 +165,4,3,-0.301129927044137,0.949586844959261,1.88928233188054 +165,5,3,-0.301129927044137,0.943744899483462,2.01036225355556 +166,1,3,-0.549765513879748,0.651497636134973,0.935561103491787 +166,2,3,-0.549765513879748,-0.129357145730795,2.34756202467047 +166,3,3,-0.549765513879748,0.637365006242933,2.98981842673778 +166,4,3,-0.549765513879748,0.966660731480892,5.43125892856486 +166,5,3,-0.549765513879748,0.133694598822538,5.07759223972584 +167,1,3,-0.78547554309208,-0.134836724965572,-1.36799223380912 +167,2,3,-0.78547554309208,0.184023568082734,-0.0674588789252142 +167,3,3,-0.78547554309208,0.221814204202511,2.78941494261896 +167,4,3,-0.78547554309208,-0.0907598902618565,1.28630104303229 +167,5,3,-0.78547554309208,-0.490928636693926,3.28879673375415 +168,1,3,-1.16481614736776,-1.05549899084636,2.30664287448066 +168,2,3,-1.16481614736776,-0.109049141845168,0.403074535883857 +168,3,3,-1.16481614736776,0.408996655515945,2.96852326627862 +168,4,3,-1.16481614736776,1.13286892015195,3.26151734891858 +168,5,3,-1.16481614736776,0.25309773918563,4.60609542882273 +169,1,3,-0.100473245130853,-0.286277055576258,0.816176439613865 +169,2,3,-0.100473245130853,0.148238163475768,1.89100786414498 +169,3,3,-0.100473245130853,1.03809923428597,3.45061203241681 +169,4,3,-0.100473245130853,0.439631515666702,4.97362624547598 +169,5,3,-0.100473245130853,0.676649197592102,6.785372217849 +170,1,3,1.52370285962906,0.928353518230501,-1.74430422590474 +170,2,3,1.52370285962906,0.857441727058135,-1.77069380443786 +170,3,3,1.52370285962906,1.95225078764784,3.5723941518199 +170,4,3,1.52370285962906,0.174810994139092,3.22889090694693 +170,5,3,1.52370285962906,1.45240210098787,5.61292746676696 +171,1,3,1.70021766980079,0.845187574622548,2.88850505078726 +171,2,3,1.70021766980079,1.60306723055171,3.48529952975244 +171,3,3,1.70021766980079,2.10398199671952,2.99813281338445 +171,4,3,1.70021766980079,1.73711534632248,5.2631280122806 +171,5,3,1.70021766980079,0.948911760532366,8.4889015355624 +172,1,3,-0.382449184151253,0.601160041563296,-0.218836631756927 +172,2,3,-0.382449184151253,0.594478731981541,0.107666949516889 +172,3,3,-0.382449184151253,0.131925459706348,1.22736418934748 +172,4,3,-0.382449184151253,0.27205811111869,4.22513044903411 +172,5,3,-0.382449184151253,0.481932581280621,4.48764564425334 +173,1,3,-0.321864791232853,-0.526916540913429,-0.919112507738525 +173,2,3,-0.321864791232853,0.82545643954509,0.575624239613491 +173,3,3,-0.321864791232853,-0.471689102491135,1.89058232409286 +173,4,3,-0.321864791232853,1.13588994941228,3.41459357670557 +173,5,3,-0.321864791232853,0.36386567246704,5.03955541787637 +174,1,3,-0.976473495882479,0.711165565116071,0.282395370482445 +174,2,3,-0.976473495882479,-0.173011423957099,-0.569514663383517 +174,3,3,-0.976473495882479,-0.518998534076677,1.38191187101233 +174,4,3,-0.976473495882479,-0.107407599484349,2.48935378518072 +174,5,3,-0.976473495882479,-0.222395331439423,4.14571722953055 +175,1,3,0.206596723904414,-0.207059321698249,-0.0363719683260104 +175,2,3,0.206596723904414,0.481350779890947,-0.334257864561706 +175,3,3,0.206596723904414,0.723281523690698,3.17349996250998 +175,4,3,0.206596723904414,1.43357721571009,2.39430459844192 +175,5,3,0.206596723904414,0.591094458526448,4.36295883976816 +176,1,3,0.0577034421995391,0.709794129416285,-0.28756468129535 +176,2,3,0.0577034421995391,-0.143545223443117,2.92356539775918 +176,3,3,0.0577034421995391,1.56256666105461,3.9682478053146 +176,4,3,0.0577034421995391,0.606339899387379,4.5254491122782 +176,5,3,0.0577034421995391,0.76336680665267,8.17539594185283 +177,1,3,0.97012801697565,0.804576615031986,0.53560539768589 +177,2,3,0.97012801697565,0.196553211150512,1.36387006814793 +177,3,3,0.97012801697565,1.00301505882641,3.51400322417612 +177,4,3,0.97012801697565,0.271445191644487,5.04448566201533 +177,5,3,0.97012801697565,1.55312861154355,6.70569729949341 +178,1,3,-0.357064593741126,-0.0607403759752695,1.01233438166498 +178,2,3,-0.357064593741126,0.490630427379729,-0.312090709939029 +178,3,3,-0.357064593741126,-0.364345488254412,2.05738386473734 +178,4,3,-0.357064593741126,0.764552669846505,1.8568801615876 +178,5,3,-0.357064593741126,1.49628986724636,4.22922510038847 +179,1,3,1.83082274441214,1.10323167318757,1.2584912282798 +179,2,3,1.83082274441214,0.995153151371863,0.976468061563311 +179,3,3,1.83082274441214,1.03251198368829,2.88884149892202 +179,4,3,1.83082274441214,1.26989041677338,4.53978057520108 +179,5,3,1.83082274441214,2.27557816245241,9.01599947501831 +180,1,3,0.155573560993865,-0.192908369096136,1.27981156464175 +180,2,3,0.155573560993865,-0.157534180273984,2.81031818521695 +180,3,3,0.155573560993865,-0.182158192618231,3.42405164561696 +180,4,3,0.155573560993865,1.35094942132599,3.96796596950746 +180,5,3,0.155573560993865,1.90194981967979,8.03537055637734 +181,1,3,-0.561729182385448,-0.716364698575066,0.0258495576976853 +181,2,3,-0.561729182385448,0.0517589100032271,-1.9223798367341 +181,3,3,-0.561729182385448,0.155757851391865,2.14638687839844 +181,4,3,-0.561729182385448,1.47783449114265,2.58274092963544 +181,5,3,-0.561729182385448,0.870450644156802,3.24356808948639 +182,1,3,0.26725208660956,-0.0491357653714034,2.49956058511792 +182,2,3,0.26725208660956,-0.210909802338806,0.886149232902366 +182,3,3,0.26725208660956,0.622334850505215,3.8931379580875 +182,4,3,0.26725208660956,1.46665374250886,5.198204176303 +182,5,3,0.26725208660956,0.627533310561888,6.03883577577174 +183,1,3,-1.58651614698834,0.0078323858732432,2.93167490126052 +183,2,3,-1.58651614698834,-0.309478667711119,0.850162920976363 +183,3,3,-1.58651614698834,-0.931464591505026,3.28271352889749 +183,4,3,-1.58651614698834,0.331128687743081,2.80293524183476 +183,5,3,-1.58651614698834,0.235317940303293,1.89763913483971 +184,1,3,0.964008293737022,0.862304156601825,1.07052054416651 +184,2,3,0.964008293737022,0.78332774517736,2.54544572781402 +184,3,3,0.964008293737022,0.760983528069229,3.13566633991018 +184,4,3,0.964008293737022,1.86056654613075,5.340943408593 +184,5,3,0.964008293737022,1.53754550750828,7.19655087170372 +185,1,3,-0.136501349380018,-0.714009827475327,0.784769499960354 +185,2,3,-0.136501349380018,0.133897812852903,0.171777702203639 +185,3,3,-0.136501349380018,-0.263791603073048,0.962395384355854 +185,4,3,-0.136501349380018,1.19789754598816,2.59010782342184 +185,5,3,-0.136501349380018,0.268244919773508,3.92974278467078 +186,1,3,-1.28961941968184,-0.918430047518358,-1.9686296021114 +186,2,3,-1.28961941968184,-1.4415577155884,-0.148603099732199 +186,3,3,-1.28961941968184,1.16603286464104,0.160275393261301 +186,4,3,-1.28961941968184,-0.420341395456616,2.6351542975303 +186,5,3,-1.28961941968184,0.219569883299303,5.01919880111458 +187,1,3,-0.50487449632609,-0.397170453608418,1.91968990836625 +187,2,3,-0.50487449632609,0.310110929892259,1.93900182773006 +187,3,3,-0.50487449632609,-0.223467929354194,1.54844471705867 +187,4,3,-0.50487449632609,0.632636963181234,2.64265112445697 +187,5,3,-0.50487449632609,1.36338592016433,5.02622184660742 +188,1,3,0.346591684139401,0.147175470379114,-0.466845097919033 +188,2,3,0.346591684139401,1.03169374286638,0.659536188793482 +188,3,3,0.346591684139401,1.17291033726791,1.44307857044969 +188,4,3,0.346591684139401,0.9003299952002,2.15240892070531 +188,5,3,0.346591684139401,0.903593984244808,5.02830703671822 +189,1,3,0.576308110605711,-0.122536463758614,-1.57705029269863 +189,2,3,0.576308110605711,0.291477322026575,-0.306041213307675 +189,3,3,0.576308110605711,0.690232742116615,2.2422184539491 +189,4,3,0.576308110605711,1.01847578790693,1.08223612517947 +189,5,3,0.576308110605711,0.711335171990724,4.07221565330838 +190,1,3,2.08095198548021,1.33707232095089,1.3473200599755 +190,2,3,2.08095198548021,1.83156228593153,2.73102211717648 +190,3,3,2.08095198548021,0.526486445244764,3.26264599133572 +190,4,3,2.08095198548021,1.87485237391011,7.60163197735244 +190,5,3,2.08095198548021,1.4436691449854,8.69741127785079 +191,1,3,0.541741765731616,0.75677092222344,-0.17741464798554 +191,2,3,0.541741765731616,0.580456407073188,-0.352770440207899 +191,3,3,0.541741765731616,1.1237012894253,0.401484430278701 +191,4,3,0.541741765731616,0.771952065397894,5.02746098105613 +191,5,3,0.541741765731616,1.86655557642009,6.30572793910101 +192,1,3,0.374552810372194,0.606532770511559,2.05554214836543 +192,2,3,0.374552810372194,-0.514822382316804,2.18945756616643 +192,3,3,0.374552810372194,0.881168519061104,4.45925069297196 +192,4,3,0.374552810372194,0.812796538462421,6.0401274280228 +192,5,3,0.374552810372194,0.790236464746359,8.97534411362399 +193,1,3,-0.547755672644997,0.195225650839263,2.65504759756614 +193,2,3,-0.547755672644997,0.0367267682650421,-0.280690334987242 +193,3,3,-0.547755672644997,0.418469306023656,0.601330235148127 +193,4,3,-0.547755672644997,0.488693531216109,5.44972918275153 +193,5,3,-0.547755672644997,0.921971026069666,5.69678613783221 +194,1,3,0.823375237959988,0.794874722936144,-0.778832178556864 +194,2,3,0.823375237959988,1.12542250133637,-0.934186219579191 +194,3,3,0.823375237959988,1.22904636185405,2.74314586857224 +194,4,3,0.823375237959988,0.528522318252197,5.13677706748988 +194,5,3,0.823375237959988,1.4961081216668,6.19996985429195 +195,1,3,-2.28077468001876,-0.895318594048709,-1.24407433196296 +195,2,3,-2.28077468001876,-1.63435720128489,-0.86850217910119 +195,3,3,-2.28077468001876,-0.465149221884744,0.311909085828801 +195,4,3,-2.28077468001876,-0.0153283011291445,-0.250171464401362 +195,5,3,-2.28077468001876,0.402100989150701,0.679870171029218 +196,1,3,-0.877192455164409,0.0199183920553034,1.12042339769811 +196,2,3,-0.877192455164409,0.309966868712776,-0.870717661006155 +196,3,3,-0.877192455164409,-0.0191782684956241,0.804081512299063 +196,4,3,-0.877192455164409,1.184332013726,2.94402306952057 +196,5,3,-0.877192455164409,1.22883637027276,4.02831372207414 +197,1,3,-1.79625296210919,-0.361051529214706,-0.343292683318492 +197,2,3,-1.79625296210919,-0.384109190615947,1.42603749350341 +197,3,3,-1.79625296210919,-0.0226322056923884,2.55701916614874 +197,4,3,-1.79625296210919,-0.279036736874818,2.26910569930322 +197,5,3,-1.79625296210919,0.909628022790928,4.30698920325096 +198,1,3,0.294599695261137,0.89036589334075,1.69884316076478 +198,2,3,0.294599695261137,0.0796835929948345,2.69317132702638 +198,3,3,0.294599695261137,0.497516125839013,6.6276099186752 +198,4,3,0.294599695261137,0.354299212915527,7.44367073594493 +198,5,3,0.294599695261137,0.387027339411681,6.31279079228627 +199,1,3,1.03393476511463,1.05710791200614,-0.973952817405197 +199,2,3,1.03393476511463,1.5307145467393,0.234826291447174 +199,3,3,1.03393476511463,0.95979715049037,0.779861823865699 +199,4,3,1.03393476511463,1.57566721611891,0.355580473898519 +199,5,3,1.03393476511463,1.95098305906965,4.97771532813293 +200,1,3,0.67406226635374,0.275277717694543,0.843334384136715 +200,2,3,0.67406226635374,1.30090857740661,1.93264815873112 +200,3,3,0.67406226635374,0.555597032912415,1.0968307169484 +200,4,3,0.67406226635374,2.22816439869584,5.46923755530594 +200,5,3,0.67406226635374,1.61344654035566,7.23989637649208 +201,1,4,-0.145619005764009,0.425062464970386,0.261616790409442 +201,2,4,-0.145619005764009,0.868817131430003,-0.445132841776166 +201,3,4,-0.145619005764009,0.492502278198704,1.83029648571645 +201,4,4,-0.145619005764009,0.630758469484849,3.30405105563127 +201,5,4,-0.145619005764009,0.776041722729689,5.63701109236231 +202,1,4,0.20848748143,-0.194711403480027,0.179745580798258 +202,2,4,0.20848748143,0.492994024084207,1.13854396901761 +202,3,4,0.20848748143,0.783119039986885,2.92293105342569 +202,4,4,0.20848748143,0.299329049970995,4.60042133367869 +202,5,4,0.20848748143,1.26220961181419,5.89206406267787 +203,1,4,-0.0189341751210595,0.882938170134449,-2.05139873048146 +203,2,4,-0.0189341751210595,1.22759454414411,-0.0627311545998119 +203,3,4,-0.0189341751210595,0.408892095900875,-0.568041233080671 +203,4,4,-0.0189341751210595,0.436565159144488,0.272986052753453 +203,5,4,-0.0189341751210595,1.0238662931341,4.38776490089903 +204,1,4,0.527436038711728,0.697663366868311,1.93523689027915 +204,2,4,0.527436038711728,0.427325742523677,0.473254757727205 +204,3,4,0.527436038711728,2.00923508548696,0.622493693881969 +204,4,4,0.527436038711728,0.971970604102201,2.53441331906684 +204,5,4,0.527436038711728,0.778763771105946,5.68427368611851 +205,1,4,1.27844865328638,0.687467032299747,-0.380514602850353 +205,2,4,1.27844865328638,0.849429647247574,-0.681233299781682 +205,3,4,1.27844865328638,1.70010462355952,1.11108413627184 +205,4,4,1.27844865328638,0.655787819387356,3.39764474726244 +205,5,4,1.27844865328638,1.88135604224227,5.64744922059397 +206,1,4,-1.18246074137794,0.170036724255962,1.13871929843077 +206,2,4,-1.18246074137794,-0.12748806549754,-0.0532917062726999 +206,3,4,-1.18246074137794,-0.370786134857948,-0.896112659770101 +206,4,4,-1.18246074137794,-0.00441465296565557,3.02106930082714 +206,5,4,-1.18246074137794,1.42894983162112,4.63504222287801 +207,1,4,-0.157505532820991,0.716516078170618,0.509385682877153 +207,2,4,-0.157505532820991,0.000137832348927613,1.72928345426769 +207,3,4,-0.157505532820991,0.544829211905254,3.3758879541191 +207,4,4,-0.157505532820991,1.06084126726245,2.44803531226303 +207,5,4,-0.157505532820991,1.39705579795745,4.77175746613998 +208,1,4,0.397900614221911,1.00807136568749,-0.252075928949645 +208,2,4,0.397900614221911,0.838451756495337,-0.268883833866314 +208,3,4,0.397900614221911,0.826190093558104,3.20343190402967 +208,4,4,0.397900614221911,0.766221557988312,3.90633552973664 +208,5,4,0.397900614221911,1.24218805277057,6.43545801727386 +209,1,4,3.08818771202226,1.18057364412943,2.18396135142987 +209,2,4,3.08818771202226,2.27702919591625,3.55491931574783 +209,3,4,3.08818771202226,1.85263702901336,5.08918598891455 +209,4,4,3.08818771202226,2.78921862657327,6.17571524563542 +209,5,4,3.08818771202226,3.2225773233352,8.73776270096561 +210,1,4,1.68702207569785,1.7993545395904,0.532838840080434 +210,2,4,1.68702207569785,0.869440033595608,2.2410028229324 +210,3,4,1.68702207569785,1.01883836380461,2.13221062747574 +210,4,4,1.68702207569785,2.00583631958716,5.87886551140786 +210,5,4,1.68702207569785,1.70231546818835,8.79615064905242 +211,1,4,-0.875883372538378,-0.978958973674517,0.581621421483876 +211,2,4,-0.875883372538378,-0.786653411189003,1.41121195690622 +211,3,4,-0.875883372538378,-0.190563082665021,2.20404293055772 +211,4,4,-0.875883372538378,0.260808890938771,2.21556675699285 +211,5,4,-0.875883372538378,0.834906311643272,3.43606019304585 +212,1,4,1.30400386151833,-0.0739311314895021,-0.27259878117807 +212,2,4,1.30400386151833,1.81479648564854,2.13416997391053 +212,3,4,1.30400386151833,0.995640877361736,1.45154720287005 +212,4,4,1.30400386151833,1.30443724412722,3.63731216528277 +212,5,4,1.30400386151833,1.13207952550597,5.59816451674992 +213,1,4,-0.23259357883221,0.543204811836342,-1.01449534225791 +213,2,4,-0.23259357883221,0.12693829181683,-1.00590686596478 +213,3,4,-0.23259357883221,-0.550437546679332,-0.857902657994927 +213,4,4,-0.23259357883221,1.08009092657631,1.48411443232596 +213,5,4,-0.23259357883221,1.28748975003943,2.58372978563663 +214,1,4,0.934509368328794,0.283756358537217,3.21736850838587 +214,2,4,0.934509368328794,0.997296492121042,3.83994823577147 +214,3,4,0.934509368328794,0.404522074604913,3.87328670453522 +214,4,4,0.934509368328794,1.75594042974006,4.3940685244154 +214,5,4,0.934509368328794,1.15095624853172,5.93523237524799 +215,1,4,-2.02633774528942,-0.627275698118266,-1.29360704342256 +215,2,4,-2.02633774528942,-0.984718100457219,-0.77486879653039 +215,3,4,-2.02633774528942,-0.797652009536068,-3.57862337525148 +215,4,4,-2.02633774528942,-0.696679896135425,-0.940107496082131 +215,5,4,-2.02633774528942,-0.600475556059633,-0.057816328034735 +216,1,4,-1.1345723514721,-0.999801486681439,-1.99926954853096 +216,2,4,-1.1345723514721,0.051632320309278,-1.30478035939822 +216,3,4,-1.1345723514721,-0.218461416657432,-0.639274587840224 +216,4,4,-1.1345723514721,0.477947371104333,0.953246898586203 +216,5,4,-1.1345723514721,0.248014439178224,2.70084584246067 +217,1,4,0.29136391143389,0.183245908076356,1.68485761265153 +217,2,4,0.29136391143389,0.395426152640086,1.93259002639756 +217,3,4,0.29136391143389,0.872056192556017,0.908055760001004 +217,4,4,0.29136391143389,0.48010831096284,2.72864700809539 +217,5,4,0.29136391143389,0.766822417416036,4.90222656954324 +218,1,4,1.02025953266783,1.18979863184446,4.13767206136758 +218,2,4,1.02025953266783,1.31261695412382,3.3885702756416 +218,3,4,1.02025953266783,1.62271221719155,3.51247762596368 +218,4,4,1.02025953266783,2.30279001427239,6.44275765130315 +218,5,4,1.02025953266783,2.15392921059779,6.075212576793 +219,1,4,0.0658993928206432,0.790306891360803,-0.482614087946492 +219,2,4,0.0658993928206432,-0.349717346583336,1.52429755080546 +219,3,4,0.0658993928206432,0.56426858103447,3.24615220616319 +219,4,4,0.0658993928206432,1.20758231685881,2.95719739567898 +219,5,4,0.0658993928206432,0.327607937927172,7.82263208692572 +220,1,4,-1.02912673591929,-1.16919501498074,-3.13433236697156 +220,2,4,-1.02912673591929,0.434877627773687,-1.23305344836268 +220,3,4,-1.02912673591929,-0.797303788823903,-0.284062193305323 +220,4,4,-1.02912673591929,0.116308946916564,1.35017095093358 +220,5,4,-1.02912673591929,0.94393107694499,1.91875502569586 +221,1,4,-0.382443187726708,0.389597842296713,0.382879340799262 +221,2,4,-0.382443187726708,0.374049508941051,0.114914620326604 +221,3,4,-0.382443187726708,-0.127663034111751,1.20459217867908 +221,4,4,-0.382443187726708,0.293992048559745,2.99629254268107 +221,5,4,-0.382443187726708,1.36640484160858,5.19669505614587 +222,1,4,-2.07378709910601,-1.2515950543621,0.958666353461258 +222,2,4,-2.07378709910601,-0.827366544533211,0.931687279406236 +222,3,4,-2.07378709910601,0.2149092035418,1.61719600463625 +222,4,4,-2.07378709910601,-0.182078896920811,1.44021079569763 +222,5,4,-2.07378709910601,-1.05195888378581,1.57467042273008 +223,1,4,0.0571434861440279,0.947769560761134,1.0358412759774 +223,2,4,0.0571434861440279,0.41621436563672,0.292865445061844 +223,3,4,0.0571434861440279,1.48618542216699,3.40291389631824 +223,4,4,0.0571434861440279,0.762338360571163,1.96188222901659 +223,5,4,0.0571434861440279,0.177984948090528,4.52859209908139 +224,1,4,-0.190737942505499,-0.738255736922229,0.127569868577206 +224,2,4,-0.190737942505499,0.372240080607862,-0.12501447773974 +224,3,4,-0.190737942505499,0.619002253283965,0.887761699211636 +224,4,4,-0.190737942505499,-0.221048847835207,2.18896719978384 +224,5,4,-0.190737942505499,1.03986148135431,5.75670165761339 +225,1,4,0.504750370322883,-0.0984297837436904,1.67458725198859 +225,2,4,0.504750370322883,1.27231230071316,2.08663858390143 +225,3,4,0.504750370322883,0.322652963868896,2.95092850709882 +225,4,4,0.504750370322883,0.478947178775107,5.43233008713638 +225,5,4,0.504750370322883,0.825625553003256,8.74656660654164 +226,1,4,0.566646966396415,0.385509168954166,0.789949604386638 +226,2,4,0.566646966396415,0.761102906211001,0.77574412438031 +226,3,4,0.566646966396415,0.864967381215401,1.58861176824269 +226,4,4,0.566646966396415,1.51851886429129,3.61972237667015 +226,5,4,0.566646966396415,1.76621774098752,4.61367138253264 +227,1,4,0.416298605248852,0.241969846324854,0.461314108917958 +227,2,4,0.416298605248852,0.307258447403572,1.02689317889856 +227,3,4,0.416298605248852,0.389077651776476,0.410468139553837 +227,4,4,0.416298605248852,1.17249774667565,3.58318359838724 +227,5,4,0.416298605248852,1.09203011758049,3.99468544614685 +228,1,4,-1.8216741221751,-0.49545657763507,-0.918010485280072 +228,2,4,-1.8216741221751,0.027308409221532,1.68188149042672 +228,3,4,-1.8216741221751,-0.594474408950622,1.80605678566885 +228,4,4,-1.8216741221751,-0.541947413609883,1.61611304815113 +228,5,4,-1.8216741221751,0.627253636608009,2.48045304876612 +229,1,4,1.95837805386369,1.21634091921974,-0.462827312292844 +229,2,4,1.95837805386369,1.74988569899565,0.945110855282057 +229,3,4,1.95837805386369,1.52757147585732,2.50011393068116 +229,4,4,1.95837805386369,1.99132635448695,4.01187377755918 +229,5,4,1.95837805386369,1.31915164171884,5.35579248615526 +230,1,4,-0.854064145631843,-1.41460473600476,0.144806847230399 +230,2,4,-0.854064145631843,0.662825868088806,1.89877543200894 +230,3,4,-0.854064145631843,-0.652182540520576,0.308075121258965 +230,4,4,-0.854064145631843,0.68245012625534,1.76009392748307 +230,5,4,-0.854064145631843,1.61509807716483,2.02400699601978 +231,1,4,-1.83315433652527,-0.751082840375815,-0.520185148961388 +231,2,4,-1.83315433652527,-0.88514688326197,1.65944375180379 +231,3,4,-1.83315433652527,-0.295660470843824,-0.907336878398132 +231,4,4,-1.83315433652527,-0.495792414559182,-0.225431070997175 +231,5,4,-1.83315433652527,0.0418227109791764,2.81883558740669 +232,1,4,1.24837686380501,0.838224876289186,-0.621704789994096 +232,2,4,1.24837686380501,2.25510434548755,0.328480973937103 +232,3,4,1.24837686380501,2.10481475648221,2.88086825041376 +232,4,4,1.24837686380501,1.42429405303522,3.36866842489618 +232,5,4,1.24837686380501,1.05793232096196,6.97140678219277 +233,1,4,2.85114953488483,1.29139647396778,0.99238604179257 +233,2,4,2.85114953488483,1.8705653471451,-0.134331908602972 +233,3,4,2.85114953488483,2.20740279929195,3.07499052105055 +233,4,4,2.85114953488483,2.15171879921461,5.39633768858733 +233,5,4,2.85114953488483,2.80830718066296,8.80251164018261 +234,1,4,-0.936743330351871,-0.0945872932981996,-0.141865793981232 +234,2,4,-0.936743330351871,0.574562882075806,0.403537407215633 +234,3,4,-0.936743330351871,0.0997238820429076,-1.1903911691781 +234,4,4,-0.936743330351871,0.361027224824763,0.762696042186893 +234,5,4,-0.936743330351871,0.557639499328201,3.13537313460174 +235,1,4,1.47580185605354,0.00281937707094781,2.6165660705684 +235,2,4,1.47580185605354,1.34854747593014,2.41643446713607 +235,3,4,1.47580185605354,1.99294917196747,2.43476543091642 +235,4,4,1.47580185605354,1.6967718166645,5.85878317716718 +235,5,4,1.47580185605354,1.95520672221328,8.07589311734043 +236,1,4,0.208747033908035,0.0745711285520664,1.31236043050198 +236,2,4,0.208747033908035,1.20431328156382,1.33371052800362 +236,3,4,0.208747033908035,-0.0523967515138719,2.31728577207176 +236,4,4,0.208747033908035,0.307677941881186,3.09254540576663 +236,5,4,0.208747033908035,0.894146130541882,2.88336258223656 +237,1,4,0.896059204631448,1.18810550586458,1.30092916076486 +237,2,4,0.896059204631448,0.454227458333857,1.52099651693863 +237,3,4,0.896059204631448,1.35758023985095,1.35051819874402 +237,4,4,0.896059204631448,0.331892873266859,5.76870679312247 +237,5,4,0.896059204631448,1.41000502507945,6.23106759672651 +238,1,4,0.113951054067604,-0.220492161787358,0.481424354940191 +238,2,4,0.113951054067604,0.811282363216797,0.725121018412195 +238,3,4,0.113951054067604,-0.34398335882458,-1.56392926823064 +238,4,4,0.113951054067604,1.53352069000688,1.92056368213871 +238,5,4,0.113951054067604,1.65603526197291,3.2588267214286 +239,1,4,0.332304485256041,1.15620286359608,-1.6681412354686 +239,2,4,0.332304485256041,0.747202621907737,-0.314622856456936 +239,3,4,0.332304485256041,0.334322005229968,-0.308578372252669 +239,4,4,0.332304485256041,1.42756911312914,2.47095762222018 +239,5,4,0.332304485256041,0.5879033667627,3.50822725475215 +240,1,4,0.00970594683895151,-0.0704452216851235,3.04122955188286 +240,2,4,0.00970594683895151,0.76395694446961,2.95416927363478 +240,3,4,0.00970594683895151,-0.046594574847339,2.45553375362412 +240,4,4,0.00970594683895151,0.769993965782161,3.94403688734056 +240,5,4,0.00970594683895151,-0.443491592376686,5.34311300206848 +241,1,4,-0.223773244924263,-0.65278284105429,-1.48828544189116 +241,2,4,-0.223773244924263,0.593489908952763,0.370330080113892 +241,3,4,-0.223773244924263,0.545104944921187,0.891140750941535 +241,4,4,-0.223773244924263,1.64379566963664,3.74751639708699 +241,5,4,-0.223773244924263,0.846644395352223,4.99132096127366 +242,1,4,0.311444780258236,1.48178803288983,-0.380767459864371 +242,2,4,0.311444780258236,1.2734006735771,0.0259808435259069 +242,3,4,0.311444780258236,-0.109772227045998,1.50016359137237 +242,4,4,0.311444780258236,0.593862883300123,2.71532805813609 +242,5,4,0.311444780258236,1.72865594619768,3.97334193486166 +243,1,4,0.341619072387833,-0.207498945762728,-0.820909073821433 +243,2,4,0.341619072387833,0.385145039448137,-0.841004612792712 +243,3,4,0.341619072387833,0.368659884620509,0.0844219376525748 +243,4,4,0.341619072387833,1.22635035866907,0.952682324592303 +243,5,4,0.341619072387833,1.76880167951173,4.90943609337553 +244,1,4,-2.04126212237833,-1.60140313192775,0.663827237102985 +244,2,4,-2.04126212237833,-0.930248302703456,1.58493805703856 +244,3,4,-2.04126212237833,0.346794972361608,0.410523261284975 +244,4,4,-2.04126212237833,-0.401524382975107,1.79390821724782 +244,5,4,-2.04126212237833,0.273694848325163,1.61289090300176 +245,1,4,-0.36739874796181,0.384491144940372,-0.570341416585646 +245,2,4,-0.36739874796181,-0.0554327743052686,1.77983314229872 +245,3,4,-0.36739874796181,1.17310166519544,1.2281890200107 +245,4,4,-0.36739874796181,1.24012454957959,2.79092718189971 +245,5,4,-0.36739874796181,0.779841451650018,4.30924212933595 +246,1,4,-0.231984127628662,0.476599317012363,0.43657726339478 +246,2,4,-0.231984127628662,0.248843711953803,0.863012870447415 +246,3,4,-0.231984127628662,0.274345464327848,1.55119128297576 +246,4,4,-0.231984127628662,0.473713731438912,3.09769523693271 +246,5,4,-0.231984127628662,0.222520430314378,3.86862426455256 +247,1,4,-0.0956398371392947,0.149491999232215,0.598097954992508 +247,2,4,-0.0956398371392947,0.022604092587618,-0.0664439657673578 +247,3,4,-0.0956398371392947,0.220212399721201,0.524357762567856 +247,4,4,-0.0956398371392947,1.00365977468176,2.46031605710149 +247,5,4,-0.0956398371392947,1.28279241725496,4.85795540887373 +248,1,4,-1.70619282588905,-0.645361101889452,1.9916344801831 +248,2,4,-1.70619282588905,-0.862927516651366,3.66791717271056 +248,3,4,-1.70619282588905,0.0127762391683118,0.609025656290565 +248,4,4,-1.70619282588905,-0.658014594679568,3.18183436799844 +248,5,4,-1.70619282588905,-0.072577383389055,2.07262299675295 +249,1,4,-0.927336330190315,0.0705125883849166,2.91410539149122 +249,2,4,-0.927336330190315,-0.151899489836396,2.18979215037809 +249,3,4,-0.927336330190315,-0.42190651916858,1.65888100186867 +249,4,4,-0.927336330190315,0.460398446536725,4.08669056464587 +249,5,4,-0.927336330190315,0.600612524859274,5.90915200490956 +250,1,4,-1.14162801235656,0.206590383641819,0.28352920602639 +250,2,4,-1.14162801235656,-0.47704255454694,1.449317489013 +250,3,4,-1.14162801235656,-0.263536574659807,-0.4116398920302 +250,4,4,-1.14162801235656,0.292921465867566,0.36941772834878 +250,5,4,-1.14162801235656,0.326599649673672,3.25456142230352 +251,1,4,-0.533133577076554,0.331363034011417,-1.68870149736653 +251,2,4,-0.533133577076554,-0.117821643463392,0.857656318646477 +251,3,4,-0.533133577076554,1.30481063021976,0.226720234297566 +251,4,4,-0.533133577076554,-0.0456515009391002,1.62908564662487 +251,5,4,-0.533133577076554,0.336734673783287,2.6538556184386 +252,1,4,-1.4242396017973,-0.616487643209825,-1.16197382550618 +252,2,4,-1.4242396017973,0.809377735235426,0.0307873435046787 +252,3,4,-1.4242396017973,0.105432878183446,-0.0608605853037751 +252,4,4,-1.4242396017973,-0.0919608429902865,2.1255165629107 +252,5,4,-1.4242396017973,1.14213687390222,1.39823536766718 +253,1,4,0.0421130196327412,1.03467783928002,-1.11463016065861 +253,2,4,0.0421130196327412,0.426069218333878,-0.766654289126376 +253,3,4,0.0421130196327412,0.434427914234819,1.22879798728331 +253,4,4,0.0421130196327412,0.956344900243424,1.95082296497428 +253,5,4,0.0421130196327412,0.995569508719923,6.3550670944955 +254,1,4,1.26480281838255,1.44835886628602,1.17559904360065 +254,2,4,1.26480281838255,0.882706330710806,1.49438786415408 +254,3,4,1.26480281838255,1.32459160552475,2.15149912950138 +254,4,4,1.26480281838255,0.784870571407219,2.86896633076918 +254,5,4,1.26480281838255,1.15274927647445,6.9490189531146 +255,1,4,-1.90012999787815,0.673194082270379,-0.096756989707168 +255,2,4,-1.90012999787815,-1.02133154445786,0.000225348399189085 +255,3,4,-1.90012999787815,-0.50678712210121,-1.02312348114859 +255,4,4,-1.90012999787815,0.542192852658531,0.997428462853665 +255,5,4,-1.90012999787815,0.273626936566653,0.92625656443509 +256,1,4,-0.252346344034527,1.03251230536201,0.683415454947388 +256,2,4,-0.252346344034527,0.161490907558304,0.168102835634015 +256,3,4,-0.252346344034527,0.498605839954082,1.01539631672537 +256,4,4,-0.252346344034527,0.110756718531093,2.9865347314332 +256,5,4,-0.252346344034527,0.798434708656316,5.63224386538034 +257,1,4,0.295978757655662,0.216684978603956,-1.77045143244746 +257,2,4,0.295978757655662,0.0586862830669549,1.44714523155237 +257,3,4,0.295978757655662,0.523459956256675,0.538358101141446 +257,4,4,0.295978757655662,1.56990095241408,2.09305803603714 +257,5,4,0.295978757655662,1.22199253431524,3.45003271456258 +258,1,4,1.45826398970798,1.73386027058223,-0.407308799164423 +258,2,4,1.45826398970798,1.61805486454142,-0.604501564966646 +258,3,4,1.45826398970798,2.15750962591565,-0.0310382347093021 +258,4,4,1.45826398970798,1.2065993232119,2.57988120880643 +258,5,4,1.45826398970798,2.3927575283945,4.378101946541 +259,1,4,-1.74927828242113,-0.9582717131597,-0.291874873747854 +259,2,4,-1.74927828242113,-0.0841825243742936,2.55602592722196 +259,3,4,-1.74927828242113,-0.66186210802338,1.08527973151208 +259,4,4,-1.74927828242113,-1.08468288544549,0.00776297528025038 +259,5,4,-1.74927828242113,0.0597058834678753,2.38968504577223 +260,1,4,-0.137485383994025,0.792215203761203,-1.10257299306267 +260,2,4,-0.137485383994025,0.0812225692570001,0.457900169155973 +260,3,4,-0.137485383994025,0.0805787039345165,1.08921496380089 +260,4,4,-0.137485383994025,0.443699607906665,3.65248584537344 +260,5,4,-0.137485383994025,0.574308585361864,2.1595938602977 +261,1,4,0.499633628174919,1.13992947289452,-1.70794926705912 +261,2,4,0.499633628174919,0.522811863197591,1.86087969706738 +261,3,4,0.499633628174919,0.56806718260539,2.17218766492301 +261,4,4,0.499633628174919,1.01730076322439,1.34763443391338 +261,5,4,0.499633628174919,1.47679969319364,2.79623267918103 +262,1,4,1.08067198678011,1.17717605152764,1.20733511818297 +262,2,4,1.08067198678011,1.1751219890645,2.17608523804642 +262,3,4,1.08067198678011,1.701187483165,3.52833996545476 +262,4,4,1.08067198678011,1.41892071374161,4.73777070820504 +262,5,4,1.08067198678011,1.7323885465477,7.84386251335008 +263,1,4,-0.56060719046659,0.105163095142194,1.2071490768394 +263,2,4,-0.56060719046659,0.0198785644593,0.831932700091331 +263,3,4,-0.56060719046659,0.449070786540603,1.66734205810187 +263,4,4,-0.56060719046659,-0.0394458634330443,2.59047103292563 +263,5,4,-0.56060719046659,0.678285785599231,3.15533257937058 +264,1,4,0.0907057898496138,-0.47278090245472,1.58095980850717 +264,2,4,0.0907057898496138,0.446766877366091,-1.38279888216449 +264,3,4,0.0907057898496138,0.544943667505921,2.71674869734128 +264,4,4,0.0907057898496138,0.76573621541547,2.96574195549868 +264,5,4,0.0907057898496138,1.41895561912996,1.95434593015983 +265,1,4,1.08467352421369,1.3456816273122,-0.708797764954616 +265,2,4,1.08467352421369,1.08450782081211,-1.78097508051042 +265,3,4,1.08467352421369,1.39961924915943,-1.79699087295665 +265,4,4,1.08467352421369,1.17295798482399,4.10392366788344 +265,5,4,1.08467352421369,1.25389946236374,3.77002388463445 +266,1,4,-0.663235486302357,-0.156636189295917,-0.913206940327043 +266,2,4,-0.663235486302357,-0.157832130555018,-1.0445091310822 +266,3,4,-0.663235486302357,1.16715836241289,0.447228752399067 +266,4,4,-0.663235486302357,0.216898343439123,1.26839928522657 +266,5,4,-0.663235486302357,0.953672436944933,2.48130106007881 +267,1,4,0.507544490493503,0.417972511745019,0.941617354075974 +267,2,4,0.507544490493503,1.18950643981523,1.9862507863618 +267,3,4,0.507544490493503,0.371223098149939,2.95823701428986 +267,4,4,0.507544490493503,1.54379395353631,4.3867244894278 +267,5,4,0.507544490493503,1.78172598951428,7.97799074265316 +268,1,4,-1.58517361075372,-0.449522195882874,0.360124536502925 +268,2,4,-1.58517361075372,-0.636906574393382,-0.0564251355860901 +268,3,4,-1.58517361075372,-0.498536311791312,-0.784646559316506 +268,4,4,-1.58517361075372,0.215602396572638,-0.0850944418558538 +268,5,4,-1.58517361075372,-0.190835436734482,1.64915421342684 +269,1,4,-0.451581849130411,0.788061274122034,0.443360546254768 +269,2,4,-0.451581849130411,-0.255544152496929,0.0991548889815735 +269,3,4,-0.451581849130411,-0.0456743309929889,-0.0154119586940922 +269,4,4,-0.451581849130411,0.863064056517142,1.38119858118668 +269,5,4,-0.451581849130411,0.375685752463198,3.70360413505207 +270,1,4,0.291875229266022,1.16560069998419,-0.0623894341601451 +270,2,4,0.291875229266022,0.478483492965743,1.60919552841901 +270,3,4,0.291875229266022,0.893078366696526,2.0977066590772 +270,4,4,0.291875229266022,1.55374186798575,2.24487890667299 +270,5,4,0.291875229266022,0.608647637970957,3.53143962307269 +271,1,4,0.0779558094282166,0.341207463349526,1.23452468056018 +271,2,4,0.0779558094282166,0.952023617349011,0.641523269725003 +271,3,4,0.0779558094282166,0.809678561125316,1.4314431131351 +271,4,4,0.0779558094282166,0.998192867745152,1.96103023715827 +271,5,4,0.0779558094282166,0.727489746127947,4.93574553674648 +272,1,4,0.302325606233118,-0.313005897459611,2.22744223449703 +272,2,4,0.302325606233118,1.26633743489874,0.947006120888524 +272,3,4,0.302325606233118,0.456127163622586,0.915729413013545 +272,4,4,0.302325606233118,0.613026738962035,2.78755041110836 +272,5,4,0.302325606233118,1.51074045322368,4.97690784001007 +273,1,4,-0.598886389459515,-1.23214633546578,2.85034802997392 +273,2,4,-0.598886389459515,0.469231509795185,2.28936950149414 +273,3,4,-0.598886389459515,0.0653801138614925,5.32178136606866 +273,4,4,-0.598886389459515,1.01017905748285,3.99470307015833 +273,5,4,-0.598886389459515,1.47174857219337,5.94702329102721 +274,1,4,-0.67930601368561,0.636870921277184,1.62418182853493 +274,2,4,-0.67930601368561,0.10978632365971,-0.016826567941179 +274,3,4,-0.67930601368561,1.28079905770633,-0.540404005824146 +274,4,4,-0.67930601368561,0.580204977086341,3.20791975425798 +274,5,4,-0.67930601368561,0.429730028989504,4.02565224395567 +275,1,4,-0.11883036286453,0.0987066122506285,0.341097128148806 +275,2,4,-0.11883036286453,0.333640883169436,0.775998225006508 +275,3,4,-0.11883036286453,0.118838622926395,1.93875270343894 +275,4,4,-0.11883036286453,0.663691666488021,2.02147912609745 +275,5,4,-0.11883036286453,0.776990997101618,3.70891660244939 +276,1,4,-1.05250893889619,0.207706706604471,1.55724848283629 +276,2,4,-1.05250893889619,-0.331494328078079,1.9362374432695 +276,3,4,-1.05250893889619,-0.319789048714261,2.05616909833095 +276,4,4,-1.05250893889619,-0.143506203445625,2.9038522125567 +276,5,4,-1.05250893889619,0.799605265472782,4.45063561169586 +277,1,4,-0.795844037443362,0.19773315538162,-1.8378169583471 +277,2,4,-0.795844037443362,0.270395158483434,0.6880310785137 +277,3,4,-0.795844037443362,0.472347160923808,1.47613775270217 +277,4,4,-0.795844037443362,-0.768088395024826,2.51168159079353 +277,5,4,-0.795844037443362,0.547199254652822,4.15777050391864 +278,1,4,0.173615046018517,0.965776968467884,1.13520464623337 +278,2,4,0.173615046018517,0.588715357259776,1.60995251889149 +278,3,4,0.173615046018517,0.503368361124506,1.16655823323351 +278,4,4,0.173615046018517,0.299150390000537,4.05486954578688 +278,5,4,0.173615046018517,0.827024303091238,4.9639610608935 +279,1,4,0.0416951645261407,0.866967622081922,1.18077004074956 +279,2,4,0.0416951645261407,0.0551413833702407,-0.0854470487913126 +279,3,4,0.0416951645261407,1.53638709831634,1.26794253836204 +279,4,4,0.0416951645261407,1.13716982550896,2.83178478428392 +279,5,4,0.0416951645261407,1.12145927780378,5.62157478688342 +280,1,4,1.7584858414703,0.539505404090267,1.42838736347038 +280,2,4,1.7584858414703,1.99518281921814,0.520484304504163 +280,3,4,1.7584858414703,1.29540090344877,3.68015674163707 +280,4,4,1.7584858414703,2.616567296328,5.95731680505034 +280,5,4,1.7584858414703,1.86573904906666,8.38399941725952 +281,1,4,0.294307873629322,0.367016285926457,0.290612600985405 +281,2,4,0.294307873629322,0.255139421662298,1.35557946058075 +281,3,4,0.294307873629322,1.89254428009209,0.47947229670969 +281,4,4,0.294307873629322,0.756565102470336,5.02955675119337 +281,5,4,0.294307873629322,0.59436204587968,5.65770391957791 +282,1,4,-0.668920427496635,-0.16090516774787,-0.0220861603980491 +282,2,4,-0.668920427496635,-0.293932941387419,-0.860657413226595 +282,3,4,-0.668920427496635,-0.617888639151153,-1.15166725712085 +282,4,4,-0.668920427496635,0.888558954552449,2.92589475136101 +282,5,4,-0.668920427496635,0.966788957178054,2.38434469294651 +283,1,4,-0.147887549805199,1.21883984593654,0.201493397033268 +283,2,4,-0.147887549805199,-0.104507144604202,1.05560905284092 +283,3,4,-0.147887549805199,0.644480146848447,2.5893567926083 +283,4,4,-0.147887549805199,0.689421659305855,2.97529728089224 +283,5,4,-0.147887549805199,0.0146694646590244,2.7385080198275 +284,1,4,1.44872204277006,1.52566993336519,-1.09833942670499 +284,2,4,1.44872204277006,1.13121617916432,-0.763523549678274 +284,3,4,1.44872204277006,1.02618553930428,2.14434055999589 +284,4,4,1.44872204277006,2.32644315791059,4.70517765580124 +284,5,4,1.44872204277006,1.6632694223249,4.67184316918207 +285,1,4,-1.73133492647527,-1.1333333954352,0.530923214738404 +285,2,4,-1.73133492647527,-0.0797624768335474,2.04554158053777 +285,3,4,-1.73133492647527,0.279531137402414,1.22563072312665 +285,4,4,-1.73133492647527,0.324317655202742,1.53333743270017 +285,5,4,-1.73133492647527,-0.0842300049235476,3.21185361312661 +286,1,4,0.573428126344241,0.794394353051903,4.01969753025436 +286,2,4,0.573428126344241,0.283988223371164,5.20447479100824 +286,3,4,0.573428126344241,0.822028322194829,4.55993273559606 +286,4,4,0.573428126344241,0.36957610586572,4.07308490588507 +286,5,4,0.573428126344241,0.461021582718634,8.45361684973381 +287,1,4,1.49761969936007,0.38082553869154,3.75268269573446 +287,2,4,1.49761969936007,1.89226501243119,2.33915199737279 +287,3,4,1.49761969936007,2.11120221547793,3.87823256197682 +287,4,4,1.49761969936007,0.575220036330622,7.54075988529905 +287,5,4,1.49761969936007,1.56730512370106,10.87439030553 +288,1,4,0.120198304783001,0.221838332294035,-0.0343789585175994 +288,2,4,0.120198304783001,0.912273335999318,0.486857046923631 +288,3,4,0.120198304783001,0.46617645674522,-0.324957673735775 +288,4,4,0.120198304783001,0.833832779999747,1.28965171934442 +288,5,4,0.120198304783001,1.97307799051121,2.86679713845684 +289,1,4,-1.21503732454083,0.242954873846912,2.83881937447877 +289,2,4,-1.21503732454083,-0.0432160852840555,0.078823737728758 +289,3,4,-1.21503732454083,-0.427710871312259,0.70638669292398 +289,4,4,-1.21503732454083,-0.556300707912749,0.928782733937543 +289,5,4,-1.21503732454083,-0.135667200525247,2.68144692998541 +290,1,4,-1.1571502810733,0.210540410190643,0.338661393530131 +290,2,4,-1.1571502810733,-0.539918911846132,1.72072318693133 +290,3,4,-1.1571502810733,0.484706650698575,1.59241521439714 +290,4,4,-1.1571502810733,-0.405062434886484,4.11562329467502 +290,5,4,-1.1571502810733,0.981369158137991,3.20289878600007 +291,1,4,-0.373521121162934,0.0230948219244378,1.26027391139345 +291,2,4,-0.373521121162934,-0.256186240139986,2.11891529051296 +291,3,4,-0.373521121162934,-0.441530085576679,1.62632162393161 +291,4,4,-0.373521121162934,0.838881189609289,2.97479271232015 +291,5,4,-0.373521121162934,0.445423620665624,5.739502917094 +292,1,4,0.893632838246333,0.649527073294073,-0.0466265337101809 +292,2,4,0.893632838246333,1.23151636406837,2.32570939009992 +292,3,4,0.893632838246333,1.67383795568239,2.78685583010959 +292,4,4,0.893632838246333,1.18427970593881,2.58308217320773 +292,5,4,0.893632838246333,1.10217795369724,4.17545088830462 +293,1,4,-0.17329984570749,-0.495703964986854,0.426598483323649 +293,2,4,-0.17329984570749,-0.20694697270213,1.43844736703838 +293,3,4,-0.17329984570749,0.545163558474171,3.17427049488256 +293,4,4,-0.17329984570749,0.401595214074252,3.6781335251007 +293,5,4,-0.17329984570749,0.177816834539147,3.23938527767826 +294,1,4,0.326497399280871,-0.13472193785092,1.25809685074472 +294,2,4,0.326497399280871,0.034022762598704,2.84557217537418 +294,3,4,0.326497399280871,0.598162799541413,1.46014716927973 +294,4,4,0.326497399280871,0.126200469466238,3.65678460464985 +294,5,4,0.326497399280871,0.164798225218632,5.61297214487013 +295,1,4,-1.69765547653391,-0.154280889366906,2.75895647414194 +295,2,4,-1.69765547653391,0.688766738833268,2.00287035833896 +295,3,4,-1.69765547653391,-0.476175631328628,2.83643637564895 +295,4,4,-1.69765547653391,-0.427259753858562,3.00758562895917 +295,5,4,-1.69765547653391,0.0358480577355416,4.42414645411934 +296,1,4,0.633650933641627,-0.821113587145336,-0.195984269286924 +296,2,4,0.633650933641627,-0.0816294051250152,0.955935691449413 +296,3,4,0.633650933641627,0.46272800410014,3.04460414246167 +296,4,4,0.633650933641627,1.24145014749022,2.25349804152533 +296,5,4,0.633650933641627,0.91046353957579,4.10649106176556 +297,1,4,0.573362320259365,0.211954671830595,-1.22089638368291 +297,2,4,0.573362320259365,0.531850561306936,-1.36805062084361 +297,3,4,0.573362320259365,0.294502809667766,-0.368782377037484 +297,4,4,0.573362320259365,1.6726363990039,3.37390164758102 +297,5,4,0.573362320259365,0.904425320734592,4.48803997278305 +298,1,4,0.777482844206898,0.532731775655578,-0.69093420369047 +298,2,4,0.777482844206898,1.23200658222154,2.17322151719419 +298,3,4,0.777482844206898,1.11553239844212,1.00522514103251 +298,4,4,0.777482844206898,1.50951732571487,2.41323059679681 +298,5,4,0.777482844206898,1.70040147775544,3.75537833523442 +299,1,4,0.662331180984266,0.822473966761673,-0.573524004218574 +299,2,4,0.662331180984266,-0.150138919613316,0.704459464206402 +299,3,4,0.662331180984266,0.912328456619516,2.77438102760412 +299,4,4,0.662331180984266,0.677139268370734,3.18581770685075 +299,5,4,0.662331180984266,1.45162176721901,3.32255926170751 +300,1,4,-1.7765642769854,0.0682655892671309,-2.41606241019692 +300,2,4,-1.7765642769854,-0.596264264781815,-2.58896938493558 +300,3,4,-1.7765642769854,0.244954817789035,-2.86925495415941 +300,4,4,-1.7765642769854,0.716148966232101,0.454508967483654 +300,5,4,-1.7765642769854,-0.508267542335298,0.917821961357774 diff --git a/benchmarks/data/twfeweights_unbalanced_panel.csv b/benchmarks/data/twfeweights_unbalanced_panel.csv new file mode 100644 index 000000000..4283b98c8 --- /dev/null +++ b/benchmarks/data/twfeweights_unbalanced_panel.csv @@ -0,0 +1,1501 @@ +"unit","period","first_treat","x1","xtv","outcome" +1,1,0,0.762280032796771,0.22170929102064,0.32572261663086 +1,2,0,0.762280032796771,1.68545437574437,0.32042267370171 +1,3,0,0.762280032796771,0.783974881767155,1.70068782526904 +1,4,0,0.762280032796771,0.151811746643227,2.97713297459989 +1,5,0,0.762280032796771,1.2007958635123,2.34523214422804 +1,6,0,0.762280032796771,1.78117297800612,3.02378983652724 +2,1,0,-0.503334604664531,0.003264242668709,1.68714543541815 +2,2,0,-0.503334604664531,-0.01160465898369,2.11519135601787 +2,3,0,-0.503334604664531,0.431326801314968,2.52345814716303 +2,4,0,-0.503334604664531,0.770960476173647,1.13187660893405 +2,5,0,-0.503334604664531,0.557599780535241,3.50475895194589 +2,6,0,-0.503334604664531,0.882451795015903,2.56076108897434 +3,1,0,0.639896047927709,-0.0821545846502019,0.575850883278317 +3,2,0,0.639896047927709,0.420510283615455,1.46869634270462 +3,3,0,0.639896047927709,1.13004442178635,1.12926487305912 +3,4,0,0.639896047927709,1.54970030934366,3.75188178767779 +3,5,0,0.639896047927709,1.33314318152428,3.63591530837704 +3,6,0,0.639896047927709,2.22273056340426,4.58792223153239 +4,1,0,0.550362540546485,0.365760271855874,1.09027190546647 +4,2,0,0.550362540546485,0.832282303943132,3.26795580759371 +4,3,0,0.550362540546485,0.308471950710698,2.58526381483487 +4,4,0,0.550362540546485,1.56094152897056,2.56528027385053 +4,5,0,0.550362540546485,0.975725496813423,4.77071046622743 +4,6,0,0.550362540546485,2.03906796710312,2.64882999446538 +5,1,0,-1.29901420371325,-0.460999451990144,-0.354880543831845 +5,2,0,-1.29901420371325,0.501202768546227,1.69427801668797 +5,3,0,-1.29901420371325,-0.71046414492934,-0.134441277863296 +5,4,0,-1.29901420371325,-0.404803176271545,0.955977315050681 +5,5,0,-1.29901420371325,1.19643256869914,0.94227680755701 +5,6,0,-1.29901420371325,0.302674962248639,0.661806868491084 +6,1,0,-0.28270754060892,0.509984546718241,-1.60615679868968 +6,2,0,-0.28270754060892,0.795840872958493,1.42057064874697 +6,3,0,-0.28270754060892,0.498743320252149,2.56986086833164 +6,4,0,-0.28270754060892,0.444741791228927,2.43785042499056 +6,5,0,-0.28270754060892,0.0659811927562748,2.03911739958511 +6,6,0,-0.28270754060892,0.434119468978059,2.66364554745197 +7,1,0,-0.228457094854756,-0.125665405042128,-1.50029270012919 +7,2,0,-0.228457094854756,0.579836444928065,-0.271374116596818 +7,3,0,-0.228457094854756,-0.125065661168389,0.134719536136669 +7,4,0,-0.228457094854756,0.540215036454454,1.27719558483555 +7,5,0,-0.228457094854756,0.78024096522406,1.91860134504533 +7,6,0,-0.228457094854756,1.08320687793625,2.14194113990727 +8,1,0,0.904243934680486,1.63054948591081,2.44848344757429 +8,2,0,0.904243934680486,1.12467480171044,1.73465701246783 +8,3,0,0.904243934680486,1.14298117599048,3.13058617549245 +8,4,0,0.904243934680486,1.34982399747455,5.14687438054722 +8,5,0,0.904243934680486,1.24058911320076,5.20947125423497 +8,6,0,0.904243934680486,1.91183461909463,6.66336910996244 +9,1,0,1.0703229218455,0.74531880575199,-1.01317106446202 +9,2,0,1.0703229218455,1.76213986405735,0.295535989411215 +9,3,0,1.0703229218455,0.996599825632608,3.73789484050547 +9,4,0,1.0703229218455,1.5717667357908,3.12292622990361 +9,5,0,1.0703229218455,1.53030746151368,3.66172079419978 +9,6,0,1.0703229218455,2.28600306767279,4.01915059847461 +10,1,0,-0.143983755489064,-0.231058747953471,0.202498691570403 +10,2,0,-0.143983755489064,-0.459455998164224,2.02623199895792 +10,3,0,-0.143983755489064,0.696408286328998,1.20596875375505 +10,4,0,-0.143983755489064,-0.00560727561101249,2.99415578809309 +10,5,0,-0.143983755489064,1.05054222823483,1.7483927621027 +10,6,0,-0.143983755489064,2.11038914740653,1.80106981071404 +11,1,0,0.586058897064396,0.547033869273745,-0.104895161161322 +11,2,0,0.586058897064396,0.782832252459021,-0.858059059928484 +11,3,0,0.586058897064396,0.979859018356124,-0.889891395153455 +11,4,0,0.586058897064396,1.13205981238157,0.418028183041052 +11,5,0,0.586058897064396,1.94312488363535,2.62673794895776 +11,6,0,0.586058897064396,1.34556107555346,1.69235327979137 +12,1,0,-0.0328133420909616,-0.718664318284347,1.01256054018714 +12,2,0,-0.0328133420909616,0.820872201858118,-0.951074969175408 +12,3,0,-0.0328133420909616,-0.123187668525215,0.899700408876431 +12,4,0,-0.0328133420909616,0.78432993049315,-0.16904574962847 +12,5,0,-0.0328133420909616,1.5789293870534,2.50565159085796 +12,6,0,-0.0328133420909616,1.13043900063265,2.18474286754862 +13,1,0,-0.261074291075106,-0.886304628526536,0.190059587043397 +13,2,0,-0.261074291075106,1.18300828926824,1.73607216665735 +13,3,0,-0.261074291075106,0.558340815315284,2.39851236830549 +13,4,0,-0.261074291075106,0.677849256703997,1.93832673654187 +13,5,0,-0.261074291075106,1.24279144956035,4.16803650110529 +13,6,0,-0.261074291075106,1.60879506475197,1.84949612465038 +14,1,0,-2.01934334017198,-0.823487619952887,0.965002660772585 +14,2,0,-2.01934334017198,-0.707032294505178,0.205238668207667 +14,3,0,-2.01934334017198,-0.61048067361937,-0.481964402432323 +14,4,0,-2.01934334017198,-0.471995841450106,0.539239702125543 +14,5,0,-2.01934334017198,0.20463530567174,0.600661309672454 +14,6,0,-2.01934334017198,0.563781281949436,0.603161656459746 +15,1,0,0.361957454931056,0.0189565591765361,-0.129555386494278 +15,2,0,0.361957454931056,0.106085329301318,0.425642861619322 +15,3,0,0.361957454931056,0.451190373474882,2.23576139017725 +15,4,0,0.361957454931056,1.28538019707699,4.29088314743058 +15,5,0,0.361957454931056,1.20237818127509,3.214356355133 +15,6,0,0.361957454931056,1.4677696966727,4.14629939829314 +16,1,0,-0.599632942668391,-0.0865504039587467,-2.49861640755116 +16,2,0,-0.599632942668391,0.730243732022659,0.509031417314836 +16,3,0,-0.599632942668391,-0.152765685660319,0.369550346158571 +16,4,0,-0.599632942668391,0.437564273403713,0.0251070431731277 +16,5,0,-0.599632942668391,1.25673745360795,-1.26356621386106 +16,6,0,-0.599632942668391,0.396948169632596,-1.19812926828942 +17,1,0,0.198694952835121,1.48026675231997,-0.0914373018488306 +17,2,0,0.198694952835121,0.602080106111401,-1.13650828313566 +17,3,0,0.198694952835121,0.66717979268474,1.28036383890753 +17,4,0,0.198694952835121,0.744336945605951,0.31249515532039 +17,5,0,0.198694952835121,0.734442176603276,3.41280747785275 +17,6,0,0.198694952835121,1.14792728063429,0.520393061796443 +18,1,0,-0.560769887816829,0.740566418840177,-1.13242281217491 +18,2,0,-0.560769887816829,-0.800802303021818,-1.39936169368382 +18,3,0,-0.560769887816829,1.64284271435801,-1.02033967144853 +18,4,0,-0.560769887816829,-0.33064398708322,0.436977600914218 +18,5,0,-0.560769887816829,1.52038443046078,1.20604277293126 +18,6,0,-0.560769887816829,1.07184454667575,0.846352514375142 +19,1,0,0.426318371985953,0.615622372338839,1.52561631350584 +19,2,0,0.426318371985953,1.1829040046622,3.60645868325585 +19,3,0,0.426318371985953,0.60962893875821,2.6921711792421 +19,4,0,0.426318371985953,1.43857945487693,3.86330852828638 +19,5,0,0.426318371985953,1.66282398015716,4.83728207603734 +19,6,0,0.426318371985953,1.40236230094667,4.44066588875079 +20,1,0,0.0661747368407101,-0.468916572476273,0.944687218273834 +20,2,0,0.0661747368407101,0.0698315654648725,2.09812105308828 +20,3,0,0.0661747368407101,1.0687390568495,3.04216485731978 +20,4,0,0.0661747368407101,1.15786295583289,3.54263125509707 +20,5,0,0.0661747368407101,1.27377826018422,1.96382561369836 +20,6,0,0.0661747368407101,0.499986760723536,3.63781888104593 +21,1,0,0.836932519114268,1.19554747601385,0.744833504737609 +21,2,0,0.836932519114268,0.859628392812328,2.36872962935447 +21,3,0,0.836932519114268,0.748417270018536,2.15303566865993 +21,4,0,0.836932519114268,1.59274446404155,3.29582632329775 +21,5,0,0.836932519114268,1.4555910021538,4.40482843122191 +21,6,0,0.836932519114268,1.58763855557056,4.8251150631344 +22,1,0,0.0941874351853353,0.466497866418425,2.31236487898151 +22,2,0,0.0941874351853353,0.261894899289606,1.20207512086356 +22,3,0,0.0941874351853353,-0.199883909412536,2.61279927072359 +22,4,0,0.0941874351853353,1.06060738068883,2.92500959946553 +22,5,0,0.0941874351853353,0.83115437138983,2.86255384244105 +22,6,0,0.0941874351853353,0.547611602018412,3.86553389086576 +23,1,0,2.24292067852269,1.65272226128883,0.512316531737125 +23,2,0,2.24292067852269,1.10093214898091,0.278294515654655 +23,3,0,2.24292067852269,1.91261347599136,1.64154746116044 +23,4,0,2.24292067852269,1.70894499719591,3.9889543067309 +23,5,0,2.24292067852269,1.88316975411293,5.14776444387616 +23,6,0,2.24292067852269,2.22298347240385,5.60218491495817 +24,1,0,-1.35767302263554,-0.532569922147083,0.854035384193373 +24,2,0,-1.35767302263554,0.293236980944925,0.859939323644371 +24,3,0,-1.35767302263554,0.400452813605349,-0.601293901404389 +24,4,0,-1.35767302263554,-0.0464606324392122,0.377478681201744 +24,5,0,-1.35767302263554,1.30355241324811,0.0307348313527933 +24,6,0,-1.35767302263554,0.640401721123196,-0.291861720912245 +25,1,0,0.536762929066239,0.494542956395155,1.13128088027847 +25,2,0,0.536762929066239,0.413659545685529,0.175996686768765 +25,3,0,0.536762929066239,0.951090791785006,0.89142621783185 +25,4,0,0.536762929066239,1.42138214987661,1.83478859337311 +25,5,0,0.536762929066239,0.197687553568442,3.82746389246305 +25,6,0,0.536762929066239,0.997008528348205,3.29347248061781 +26,1,0,0.080424492077766,0.55057496426079,-1.57794939371139 +26,2,0,0.080424492077766,0.857425330645063,0.204173346072951 +26,3,0,0.080424492077766,0.435846052366296,0.467273164232521 +26,4,0,0.080424492077766,0.984228608534135,-0.358590158898051 +26,5,0,0.080424492077766,1.72601324978517,-0.249224951584627 +26,6,0,0.080424492077766,1.09283259694636,2.05876710749068 +27,1,0,-0.661809943463178,-0.351151472617987,1.3332812259713 +27,2,0,-0.661809943463178,-0.769094233439663,1.02937579325852 +27,3,0,-0.661809943463178,0.529301492309873,0.670624009578854 +27,4,0,-0.661809943463178,0.340211950424195,2.83276083639353 +27,5,0,-0.661809943463178,0.672804544960685,1.33367085251827 +27,6,0,-0.661809943463178,0.836997883685823,1.64463392166496 +28,1,0,1.18763780487985,0.857112953615082,0.918371137986335 +28,2,0,1.18763780487985,0.609455036547718,-0.1953916519845 +28,3,0,1.18763780487985,0.957415327060119,1.64710330460386 +28,4,0,1.18763780487985,1.70678167693069,2.82206969876212 +28,5,0,1.18763780487985,2.28994975704781,2.81441799704668 +28,6,0,1.18763780487985,2.70532991473682,6.64412995317486 +29,1,0,0.171569793047149,-0.398831901085214,0.103682763935853 +29,2,0,0.171569793047149,0.185275047950682,0.183298060409325 +29,3,0,0.171569793047149,1.16669238180421,1.92277150882252 +29,4,0,0.171569793047149,1.12694852976321,1.54493766913653 +29,5,0,0.171569793047149,1.04381631290082,4.12170013580083 +29,6,0,0.171569793047149,1.43494779620057,2.35016615646919 +30,1,0,0.214899638814364,0.990047580435093,0.0329317907015836 +30,2,0,0.214899638814364,0.00318616298013463,3.1538826374806 +30,3,0,0.214899638814364,-0.146108580250217,1.40058443711373 +30,4,0,0.214899638814364,0.538522428274927,3.45129621770057 +30,5,0,0.214899638814364,1.52956640764869,4.19427236133207 +30,6,0,0.214899638814364,1.99778295611785,3.56034464827219 +31,1,0,2.32254390100494,1.15644468484223,2.77259843088733 +31,2,0,2.32254390100494,1.7483266658964,2.02564757997795 +31,3,0,2.32254390100494,2.2699980030887,1.64606771516423 +31,4,0,2.32254390100494,2.71526529576935,3.37656661650927 +31,5,0,2.32254390100494,2.46194873478394,6.10188976491978 +31,6,0,2.32254390100494,2.97510578028662,8.52554433662228 +32,1,0,-0.169502937020473,0.693215490975842,2.05141406053865 +32,2,0,-0.169502937020473,0.468165521466207,4.61437618451743 +32,3,0,-0.169502937020473,0.205806456493878,1.11339736229636 +32,4,0,-0.169502937020473,1.25812023117349,4.1166099867286 +32,5,0,-0.169502937020473,1.44654158298815,3.40523553515562 +32,6,0,-0.169502937020473,1.31004230358703,1.01835012296617 +33,1,0,0.552770969821111,-0.306006044577907,1.25412066983468 +33,2,0,0.552770969821111,0.995602390305268,1.80917798576773 +33,3,0,0.552770969821111,1.01655508934542,0.757291376613842 +33,4,0,0.552770969821111,1.1036479557666,2.96417691332605 +33,5,0,0.552770969821111,1.45864948387186,2.46197479844314 +33,6,0,0.552770969821111,0.859567156764491,5.97071719266024 +34,1,0,0.481846461393583,0.269302303860423,0.526376540360734 +34,2,0,0.481846461393583,0.0461613847557073,2.19364928878128 +34,3,0,0.481846461393583,1.12068944272412,2.44731864429638 +34,4,0,0.481846461393583,0.783270153094354,2.52781800340864 +34,5,0,0.481846461393583,1.97211005145252,3.13973621130369 +34,6,0,0.481846461393583,1.03839429696509,2.94271775797344 +35,1,0,0.180785291726702,0.590217475976332,2.01882790659905 +35,2,0,0.180785291726702,0.408881961264579,4.08885751899202 +35,3,0,0.180785291726702,0.581997504350181,3.11714635871518 +35,4,0,0.180785291726702,1.49965041737788,5.38773518005293 +35,5,0,0.180785291726702,2.02563496208288,5.9748136314136 +35,6,0,0.180785291726702,2.28603451086318,2.7474872302793 +36,1,0,0.937530562434575,0.211835239755195,0.264811770320571 +36,2,0,0.937530562434575,0.97223160253963,3.72358171039763 +36,3,0,0.937530562434575,2.03844428066716,2.52837086994306 +36,4,0,0.937530562434575,1.35584019846206,3.46668468173737 +36,5,0,0.937530562434575,2.31452132526835,4.30663345338415 +36,6,0,0.937530562434575,1.52220257385175,5.90295382804056 +37,1,0,0.605297706819014,1.33501016334775,-1.22155851591522 +37,2,0,0.605297706819014,0.273646824543883,0.94685268474313 +37,3,0,0.605297706819014,1.18861405551878,-1.06749360477619 +37,4,0,0.605297706819014,0.430022662342312,2.49683112338337 +37,5,0,0.605297706819014,0.980782441057555,2.87859515158806 +37,6,0,0.605297706819014,1.68491715613623,3.18312049166695 +38,1,0,-0.150046075996628,0.907283121370997,-0.269990850232008 +38,2,0,-0.150046075996628,0.578967394515049,-0.886845866605914 +38,3,0,-0.150046075996628,1.31322956121607,2.17397969319074 +38,4,0,-0.150046075996628,0.770060624548639,0.629084998485956 +38,5,0,-0.150046075996628,0.858759768922271,0.248192249762817 +38,6,0,-0.150046075996628,1.31157900399226,3.13842264488913 +39,1,0,-1.53940169474244,0.386233806928156,-0.523153781143094 +39,2,0,-1.53940169474244,-0.184038667457276,-1.58742257093027 +39,3,0,-1.53940169474244,-0.973488223644379,-0.749417744231022 +39,4,0,-1.53940169474244,0.510965070488413,-0.735701532265713 +39,5,0,-1.53940169474244,0.766370381872722,-1.75342467837839 +39,6,0,-1.53940169474244,0.687218453403146,0.252169837110935 +40,1,0,0.391843330512968,0.704055957164002,0.409086287780538 +40,2,0,0.391843330512968,0.519946271805668,-1.89191667178992 +40,3,0,0.391843330512968,0.432251147082645,0.648330099168622 +40,4,0,0.391843330512968,1.21713541238615,1.31189584887626 +40,5,0,0.391843330512968,1.27472438671965,0.103667887130483 +40,6,0,0.391843330512968,1.66786350472797,0.685042322957091 +41,1,0,-0.0373260431491352,0.0317324594708311,-0.747942068188531 +41,2,0,-0.0373260431491352,0.558761507456241,-1.19204731827254 +41,3,0,-0.0373260431491352,0.627774174828559,-0.00444322772313305 +41,4,0,-0.0373260431491352,1.22057537184321,0.0157895549428625 +41,5,0,-0.0373260431491352,-0.13463140571047,2.03834883872972 +41,6,0,-0.0373260431491352,1.26326673119073,2.38013129655104 +42,1,0,0.827626116517122,0.619582666001477,2.05284651791961 +42,2,0,0.827626116517122,0.728960630288116,2.63692144330927 +42,3,0,0.827626116517122,1.51573241849325,2.82734402300804 +42,4,0,0.827626116517122,2.32134629984684,4.70066367856648 +42,5,0,0.827626116517122,1.26080310077716,2.71057083119057 +42,6,0,0.827626116517122,1.56560512998824,4.83856397972047 +43,1,0,-0.665331979033663,-0.225102401789033,2.56456608500982 +43,2,0,-0.665331979033663,-0.553172893008135,4.61457088107317 +43,3,0,-0.665331979033663,0.972861442802118,1.82443898633191 +43,4,0,-0.665331979033663,0.32326929160598,2.64950447327976 +43,5,0,-0.665331979033663,0.306489791796068,2.60561890553988 +43,6,0,-0.665331979033663,1.03909552234328,3.64564011990368 +44,1,0,0.942288464073336,0.00874783812863711,0.056977549744632 +44,2,0,0.942288464073336,0.596281511258369,1.12468821275525 +44,3,0,0.942288464073336,1.39468851953371,2.90909190594441 +44,4,0,0.942288464073336,2.11675576968934,4.23166721482138 +44,5,0,0.942288464073336,1.58330641741654,5.43308550177595 +44,6,0,0.942288464073336,1.16350891633872,4.71261506783713 +45,1,0,-0.0809416298757052,0.807692153394734,1.0789586456094 +45,2,0,-0.0809416298757052,0.752887274713188,0.785720766407489 +45,3,0,-0.0809416298757052,1.24054098972764,2.55331843603172 +45,4,0,-0.0809416298757052,1.34412295664021,2.48536763140985 +45,5,0,-0.0809416298757052,-0.0307181568992582,1.1879236197245 +45,6,0,-0.0809416298757052,1.78022693303965,3.71848890245285 +46,1,0,0.521163361059873,-0.723702502547912,1.09867938696003 +46,2,0,0.521163361059873,1.1499608686645,3.55842561079306 +46,3,0,0.521163361059873,1.15232123544905,3.17634759665484 +46,4,0,0.521163361059873,1.42926586195998,3.39003436513758 +46,5,0,0.521163361059873,0.879129584097901,2.80180340599266 +46,6,0,0.521163361059873,0.914156746191509,2.46863004623085 +47,1,0,1.61466175286344,0.643006559192123,0.836204718894284 +47,2,0,1.61466175286344,1.13661030532442,2.30379657813442 +47,3,0,1.61466175286344,1.19295732982162,2.64086502103959 +47,4,0,1.61466175286344,0.239328559267526,3.65153083011424 +47,5,0,1.61466175286344,1.68375842695897,5.33901616867935 +47,6,0,1.61466175286344,2.02785308242104,4.85439287872028 +48,1,0,-1.7565659395326,-0.218238640820971,1.78646195020932 +48,2,0,-1.7565659395326,-0.969625592296388,0.784269865689745 +48,3,0,-1.7565659395326,-0.419721278011531,2.20671186654619 +48,4,0,-1.7565659395326,-0.0179174885255089,0.521674280505722 +48,5,0,-1.7565659395326,0.0262037737083689,0.883000983935286 +48,6,0,-1.7565659395326,0.434997896953442,1.76393693136589 +49,1,0,0.0236704073602128,0.342406228293342,-0.0627973687877335 +49,2,0,0.0236704073602128,1.339042197954,0.510408069568147 +49,3,0,0.0236704073602128,0.396305118083889,0.128070410293366 +49,4,0,0.0236704073602128,0.301229123953579,2.34822787611627 +49,5,0,0.0236704073602128,1.84984351148937,2.02095085612302 +49,6,0,0.0236704073602128,1.14237872441368,3.88976774422889 +50,1,0,-1.05243771447958,0.290206750660849,-0.27631290661132 +50,2,0,-1.05243771447958,-0.331976147509505,0.0784093174185179 +50,3,0,-1.05243771447958,0.644389922826399,1.85558157163888 +50,4,0,-1.05243771447958,-0.0363544318974775,1.14349827947337 +50,5,0,-1.05243771447958,-0.63902084988463,0.0282478533308865 +50,6,0,-1.05243771447958,0.277763327171488,1.60530873660652 +51,1,0,1.41893961966826,1.72807036319162,1.23790039735326 +51,2,0,1.41893961966826,1.68895995675209,2.07276778859 +51,3,0,1.41893961966826,0.971263584985605,3.6394918544787 +51,4,0,1.41893961966826,1.11899195431195,3.41900141304584 +51,5,0,1.41893961966826,1.87654473407966,4.68940054213404 +51,6,0,1.41893961966826,3.04105530464514,4.8924758822659 +52,1,0,-1.1910633756242,0.200524510461652,-2.7265059697081 +52,2,0,-1.1910633756242,-1.2115909345573,-1.40963894581555 +52,3,0,-1.1910633756242,0.386175892807225,-1.47451140247512 +52,4,0,-1.1910633756242,1.02789828620209,0.146871306849964 +52,5,0,-1.1910633756242,-0.273264466564166,-0.0141430110384286 +52,6,0,-1.1910633756242,0.578879852223559,-1.17416016414042 +53,1,0,0.195514781381017,-0.742554152921851,2.23579400276791 +53,2,0,0.195514781381017,0.23337503458647,1.10762600155431 +53,3,0,0.195514781381017,0.263301882945963,2.51597948959044 +53,4,0,0.195514781381017,-0.199791624518697,3.42840858712109 +53,5,0,0.195514781381017,1.60088371927206,1.87400311570008 +53,6,0,0.195514781381017,1.23132242768948,2.10936229576561 +54,1,0,0.111145488608917,0.41248587564804,-1.23962257171445 +54,2,0,0.111145488608917,0.650566893014723,0.757366835792272 +54,3,0,0.111145488608917,0.638921948130227,-0.317803478051985 +54,4,0,0.111145488608917,-0.29731490602435,0.682731490142251 +54,5,0,0.111145488608917,1.38608083061868,1.22724299967546 +54,6,0,0.111145488608917,1.38422425480489,3.12874530862489 +55,1,0,0.666862478709229,1.66603184966515,-0.96199867245643 +55,2,0,0.666862478709229,0.633112902252682,-1.17037332152456 +55,3,0,0.666862478709229,0.343015660916893,0.874503842048745 +55,4,0,0.666862478709229,0.770817827624448,-0.0109943396516543 +55,5,0,0.666862478709229,0.964533553442544,1.71046636135371 +55,6,0,0.666862478709229,1.76201281537328,1.8951412953794 +56,1,0,-1.77404829500802,-1.40580752868787,1.39863275680412 +56,2,0,-1.77404829500802,0.870111926030201,-0.982599170611262 +56,3,0,-1.77404829500802,-0.830299171612467,-0.778719888508792 +56,4,0,-1.77404829500802,0.334381341446428,2.10682625136074 +56,5,0,-1.77404829500802,0.0402025096737488,0.21272995862292 +56,6,0,-1.77404829500802,-0.102134961440965,-0.482035354350842 +57,1,0,-1.69162901504091,-0.937011635623388,-1.822960278522 +57,2,0,-1.69162901504091,-0.854090558628001,0.246599401891903 +57,3,0,-1.69162901504091,0.306483912711003,-1.10550309471832 +57,4,0,-1.69162901504091,-0.375186713716243,-0.403217811110663 +57,5,0,-1.69162901504091,-0.0372694866470827,-1.2397278955316 +57,6,0,-1.69162901504091,0.042822212170315,-1.72546983425334 +58,1,0,-0.849031394079241,-0.365462147353985,-0.312601656922766 +58,2,0,-0.849031394079241,-0.0832361702771065,0.103741778870931 +58,3,0,-0.849031394079241,0.142581169967471,-0.482286945873843 +58,4,0,-0.849031394079241,0.460384860409781,1.40366770538517 +58,5,0,-0.849031394079241,0.996609986700584,0.871624823978852 +58,6,0,-0.849031394079241,0.85264265006985,1.89123646594172 +59,1,0,1.21025798910521,1.04847972204707,-0.575583913121781 +59,2,0,1.21025798910521,1.38937084327584,1.37186521680745 +59,3,0,1.21025798910521,1.2547790752355,0.529851722231607 +59,4,0,1.21025798910521,1.17327232495747,1.47840784888763 +59,5,0,1.21025798910521,1.172880927698,3.91882955565966 +59,6,0,1.21025798910521,1.39574007441178,2.28500196266698 +60,1,0,0.789386089516239,0.179951915237443,0.435733939422799 +60,2,0,0.789386089516239,0.438493621227776,1.47709316141718 +60,3,0,0.789386089516239,1.21539128818204,3.37967447648071 +60,4,0,0.789386089516239,1.76191512346368,2.94762787653769 +60,5,0,0.789386089516239,1.35758016131045,2.48842314882934 +60,6,0,0.789386089516239,1.50244938572149,4.71258184189825 +61,1,0,1.15707226166531,0.695992482122723,0.366873356023154 +61,2,0,1.15707226166531,1.30843852180442,0.25928295987725 +61,3,0,1.15707226166531,1.23140830848896,0.670889744195337 +61,4,0,1.15707226166531,1.52680662397394,3.10031216794104 +61,5,0,1.15707226166531,1.97553314383926,4.52201393669106 +61,6,0,1.15707226166531,1.89587096185583,4.17000041602687 +62,1,0,-0.344242039937823,-0.012607129644549,1.53095259638453 +62,2,0,-0.344242039937823,-0.190659504558242,2.99016158987811 +62,3,0,-0.344242039937823,0.129929099988597,2.55026017406343 +62,4,0,-0.344242039937823,0.75025985280716,0.871172605987694 +62,5,0,-0.344242039937823,-0.088414077084213,2.66185824077775 +62,6,0,-0.344242039937823,1.68941056098916,2.3342515589607 +63,1,0,-0.520156154475325,0.115284773651995,-0.554998777313392 +63,2,0,-0.520156154475325,0.380215358847244,0.198926509461979 +63,3,0,-0.520156154475325,-0.045325702695873,0.599340066190899 +63,4,0,-0.520156154475325,0.403372511379866,3.05428355693047 +63,5,0,-0.520156154475325,0.339830867502976,2.03433756350707 +63,6,0,-0.520156154475325,0.952544521227367,1.17519783476071 +64,1,0,0.203435284900822,0.515493855780151,-0.749085263516488 +64,2,0,0.203435284900822,0.746187070355182,2.1052938488016 +64,3,0,0.203435284900822,0.374895246973636,1.58265151593793 +64,4,0,0.203435284900822,0.506751948936316,2.14495255559874 +64,5,0,0.203435284900822,1.51739942863017,0.95825975606708 +64,6,0,0.203435284900822,1.09147631899005,3.41111423514602 +65,1,0,-0.55341970038927,-0.131380062378949,-1.32195592808742 +65,2,0,-0.55341970038927,-0.241965241577032,-1.02423916263361 +65,3,0,-0.55341970038927,1.15300408189915,-0.535841679982353 +65,4,0,-0.55341970038927,0.0532746291132736,-0.968231722746471 +65,5,0,-0.55341970038927,0.745558003100886,0.832777050424564 +65,6,0,-0.55341970038927,-0.00751406091164042,1.96077688545181 +66,1,0,1.16897479134018,0.184878831174988,-1.28620870245401 +66,2,0,1.16897479134018,0.987928053916087,0.166753297701763 +66,3,0,1.16897479134018,1.27696264162646,1.68247659314794 +66,4,0,1.16897479134018,1.55536067948273,3.87746819914394 +66,5,0,1.16897479134018,2.15354648101136,4.79993387221023 +66,6,0,1.16897479134018,1.71085429487955,4.04950544043591 +67,1,0,-0.311033874726052,0.0263159579110574,0.871500540829715 +67,2,0,-0.311033874726052,0.213806039758833,1.10566090191848 +67,3,0,-0.311033874726052,1.16580579545896,2.62828302528987 +67,4,0,-0.311033874726052,0.941962746311597,1.57046531053 +67,5,0,-0.311033874726052,0.185785078920236,1.0287988657371 +67,6,0,-0.311033874726052,0.0881200987592246,4.48340110608756 +68,1,0,0.609445260914666,0.160720456451015,-0.875045477717211 +68,2,0,0.609445260914666,0.398587442880102,1.97447918279283 +68,3,0,0.609445260914666,0.855870863727989,1.76870737136006 +68,4,0,0.609445260914666,1.43803622432617,3.07264275168721 +68,5,0,0.609445260914666,1.53294248674142,3.1268484428726 +68,6,0,0.609445260914666,2.30216153797432,4.90626806484531 +69,1,0,-1.70140877068258,-0.772137510934144,-3.67033050406739 +69,2,0,-1.70140877068258,-0.0887373122839057,0.00741645039112426 +69,3,0,-1.70140877068258,0.23462709293423,-0.35143639642592 +69,4,0,-1.70140877068258,-0.0133450385366955,-3.11261503664916 +69,5,0,-1.70140877068258,0.647212644285179,-1.22067385183629 +69,6,0,-1.70140877068258,-0.318090934527394,-1.7703701960703 +70,1,0,-0.701528235373902,0.419756517766687,-0.392170361021321 +70,2,0,-0.701528235373902,1.00520991718909,-1.24465679664118 +70,3,0,-0.701528235373902,0.397714193248576,1.73346811162478 +70,4,0,-0.701528235373902,0.141784792120088,2.62165510171308 +70,5,0,-0.701528235373902,-0.229555991095407,1.66547134257899 +70,6,0,-0.701528235373902,0.457270807196878,3.5583092350656 +71,1,0,0.505874716504533,0.0328685484582639,1.22528702490137 +71,2,0,0.505874716504533,1.59522458519428,2.23505630653046 +71,3,0,0.505874716504533,1.76396196449465,4.50976634967245 +71,4,0,0.505874716504533,2.02663440047412,5.34020271673294 +71,5,0,0.505874716504533,1.16929077274353,5.67948718050403 +71,6,0,0.505874716504533,0.987516672556885,4.01479530785995 +72,1,0,-0.844325706369891,-0.160769095993306,1.77828123515083 +72,2,0,-0.844325706369891,-0.626002441128783,2.86943080366521 +72,3,0,-0.844325706369891,0.624503195437875,1.13819663453126 +72,4,0,-0.844325706369891,0.4128826402263,2.34116236266462 +72,5,0,-0.844325706369891,0.64321904824842,6.00376292914209 +72,6,0,-0.844325706369891,1.30388473771754,4.39817205983043 +73,1,0,0.35463969852429,0.175853581251931,1.54901516888919 +73,2,0,0.35463969852429,0.255347723083929,3.75759086722241 +73,3,0,0.35463969852429,0.842048899742575,2.74674519300349 +73,4,0,0.35463969852429,0.843858032144758,3.48463769028479 +73,5,0,0.35463969852429,2.02272986936328,2.63274428096982 +73,6,0,0.35463969852429,1.756515220362,3.36491283175246 +74,1,0,0.634049900150801,1.06843050846366,0.631806643103645 +74,2,0,0.634049900150801,1.09459023941769,1.10185630635582 +74,3,0,0.634049900150801,1.16047718202649,2.21277078081482 +74,4,0,0.634049900150801,0.194209434370495,4.49679293334212 +74,5,0,0.634049900150801,1.57111541788119,3.78301141707104 +74,6,0,0.634049900150801,1.8709185053691,3.83801701206847 +75,1,0,-0.144614154426945,-0.1812136320715,-1.50303099629943 +75,2,0,-0.144614154426945,0.0812045686177193,-0.30813252539654 +75,3,0,-0.144614154426945,0.592606669784812,-0.87606327177109 +75,4,0,-0.144614154426945,2.21284582539953,1.49034525451696 +75,5,0,-0.144614154426945,0.89070743973123,1.99305042503676 +75,6,0,-0.144614154426945,1.22531924408363,1.04904749337575 +76,1,0,-0.965620194716236,-0.416127905137339,-0.767970743384194 +76,2,0,-0.965620194716236,0.895930664804138,-0.185344950505385 +76,3,0,-0.965620194716236,0.77433449793566,0.860449806370312 +76,4,0,-0.965620194716236,0.70085446817956,1.07172526057056 +76,5,0,-0.965620194716236,1.05852067016297,0.51916850372457 +76,6,0,-0.965620194716236,0.638859520759945,1.33787453176867 +77,1,0,-1.86580715673843,-0.391278486536145,-0.0920440713171948 +77,2,0,-1.86580715673843,-1.3757502672685,-0.560366699602578 +77,3,0,-1.86580715673843,0.308335534756901,-0.371286061286313 +77,4,0,-1.86580715673843,0.345233743814379,-0.20407441671565 +77,5,0,-1.86580715673843,0.381411993188402,-1.14332176885771 +77,6,0,-1.86580715673843,-0.584252643983715,-1.27669748966723 +78,1,0,-1.36945985986056,-0.317388985722178,0.757961675843857 +78,2,0,-1.36945985986056,-0.931319605141151,-1.47231495687832 +78,3,0,-1.36945985986056,0.64939703346783,0.947760079117256 +78,4,0,-1.36945985986056,0.141848675286346,0.427700713693576 +78,5,0,-1.36945985986056,0.645979505797502,0.123611381550555 +78,6,0,-1.36945985986056,0.853461304724385,-1.30724575909746 +79,1,0,-1.4085132336069,-1.4450148783583,0.727494818634948 +79,2,0,-1.4085132336069,-0.599025158604699,0.291135574546722 +79,3,0,-1.4085132336069,-0.406989701850758,0.882944383939482 +79,4,0,-1.4085132336069,-0.0202306741507156,0.290743072184474 +79,5,0,-1.4085132336069,0.418122082147065,-0.554654948017118 +79,6,0,-1.4085132336069,0.161492862233551,-0.104305091084399 +80,1,0,1.0465570019994,0.83467245617602,1.79474289967135 +80,2,0,1.0465570019994,0.112243844479774,2.42789304208095 +80,3,0,1.0465570019994,0.333555656213314,4.11924338896377 +80,4,0,1.0465570019994,1.01788286394161,3.88357495448787 +80,5,0,1.0465570019994,1.53660910957585,4.5182271100794 +80,6,0,1.0465570019994,1.62573779201925,6.09413820742505 +81,1,0,-1.41453829223088,-0.422101284770018,0.769101887017708 +81,2,0,-1.41453829223088,-0.315168451954516,0.729566856970148 +81,3,0,-1.41453829223088,-0.272040188359851,-0.671347616711595 +81,4,0,-1.41453829223088,0.183840000922275,-0.614516066633574 +81,5,0,-1.41453829223088,0.299215308065219,0.0802272383475152 +81,6,0,-1.41453829223088,0.732800101297085,0.510417085488192 +82,1,0,0.160539536166133,0.959530186512375,-0.215930107181071 +82,2,0,0.160539536166133,-0.326281366208948,1.07948246934508 +82,3,0,0.160539536166133,0.324000083908395,1.52302152579562 +82,4,0,0.160539536166133,-0.258160129826018,1.74745221913454 +82,5,0,0.160539536166133,1.46023105551091,2.33039670539431 +82,6,0,0.160539536166133,0.788817140506763,2.32858808738527 +83,1,0,-0.567801186208016,0.231833577654445,1.9093919950695 +83,2,0,-0.567801186208016,-0.406252558339809,1.71838187145903 +83,3,0,-0.567801186208016,0.500105335834885,0.424872683579041 +83,4,0,-0.567801186208016,1.6055462707676,1.14871256154913 +83,5,0,-0.567801186208016,1.27321097698752,2.60872052340111 +83,6,0,-0.567801186208016,0.495575788605087,1.73434907598337 +84,1,0,-0.782213005252933,-1.24210551325893,0.62824879952858 +84,2,0,-0.782213005252933,0.998799311582334,-0.285355853362829 +84,3,0,-0.782213005252933,0.152353660632812,1.08574270218729 +84,4,0,-0.782213005252933,0.437770170026559,0.992180904182235 +84,5,0,-0.782213005252933,0.555952746039179,2.04941499620541 +84,6,0,-0.782213005252933,0.890162464146768,-0.894620656851764 +85,1,0,-0.487130227305875,0.47292213649988,-0.82059178734108 +85,2,0,-0.487130227305875,0.492301356302156,0.262528389117469 +85,3,0,-0.487130227305875,1.29009677447065,1.17301382240356 +85,4,0,-0.487130227305875,1.43591257033689,2.60956505836633 +85,5,0,-0.487130227305875,1.82677210590248,0.693530436043082 +85,6,0,-0.487130227305875,1.32378375915425,1.21715970534733 +86,1,0,-0.384373119535855,-0.170904424548637,0.498232930318139 +86,2,0,-0.384373119535855,-0.295740805748663,0.301854807254213 +86,3,0,-0.384373119535855,0.578383091390104,0.695014709575832 +86,4,0,-0.384373119535855,0.528019648180887,-0.737125962869525 +86,5,0,-0.384373119535855,1.01985265304516,-0.0830034763200127 +86,6,0,-0.384373119535855,1.67834611857395,2.49504179213634 +87,1,0,0.408020963230945,0.962156203251732,1.94439889343524 +87,2,0,0.408020963230945,0.99999055549027,1.16647176165158 +87,3,0,0.408020963230945,0.970786735481066,1.69236048882497 +87,4,0,0.408020963230945,1.41338326575805,4.12905636501844 +87,5,0,0.408020963230945,1.49273145808118,2.17402526741817 +87,6,0,0.408020963230945,1.37163125752385,5.37627582349371 +88,1,0,0.599584079099597,0.730924181364043,-1.31376569334669 +88,2,0,0.599584079099597,1.44699595117378,-0.623281047649964 +88,3,0,0.599584079099597,0.28253615169462,-0.693242466503484 +88,4,0,0.599584079099597,1.16021618973925,1.01881668516363 +88,5,0,0.599584079099597,0.964519264352473,1.21574324307197 +88,6,0,0.599584079099597,1.84858002344773,2.57058311503664 +89,1,0,-0.311048261021212,0.172929318481353,2.7900643892079 +89,2,0,-0.311048261021212,0.913627245775031,1.24443702713629 +89,3,0,-0.311048261021212,-0.781248428169652,1.03938341222443 +89,4,0,-0.311048261021212,0.602717775024287,0.461064451496916 +89,5,0,-0.311048261021212,0.77172145901647,2.32861716047523 +89,6,0,-0.311048261021212,0.875048468268877,4.46406247718196 +90,1,0,-0.812370112719702,0.508688758563729,0.663812620865317 +90,2,0,-0.812370112719702,0.0978888128213179,0.604934747365124 +90,3,0,-0.812370112719702,-0.227169219633528,0.483064206170982 +90,4,0,-0.812370112719702,-0.0789620142453338,1.30016784410719 +90,5,0,-0.812370112719702,0.474031207045202,-0.35852658877042 +90,6,0,-0.812370112719702,0.362037836323042,2.62002321031834 +91,1,0,-0.835160216707722,0.0821439607244709,-1.31297017293814 +91,2,0,-0.835160216707722,-0.215052170622971,-0.00952574873257295 +91,3,0,-0.835160216707722,-0.515795699189277,0.93911380476138 +91,4,0,-0.835160216707722,0.0994435769261413,0.29534073274764 +91,5,0,-0.835160216707722,0.640632783084201,0.631494983316459 +91,6,0,-0.835160216707722,0.444237884051304,-0.614904284648697 +92,1,0,-0.0347979035615357,-0.425071253260572,-0.276129691013229 +92,2,0,-0.0347979035615357,-0.00558881648247656,2.6914081615219 +92,3,0,-0.0347979035615357,0.12042472026752,0.139596914798406 +92,4,0,-0.0347979035615357,0.19294717753885,-0.0358857413034133 +92,5,0,-0.0347979035615357,0.961964380434135,2.00578635972552 +92,6,0,-0.0347979035615357,1.17135420091513,2.59638281957783 +93,1,0,-0.875486806182028,0.130202709056072,0.0515320316326158 +93,2,0,-0.875486806182028,-1.41949223413023,0.503122856462025 +93,3,0,-0.875486806182028,-0.109791084167903,-1.1180468841845 +93,4,0,-0.875486806182028,0.659965080718873,-0.368352880340779 +93,5,0,-0.875486806182028,0.067354181000394,0.0520547024496831 +93,6,0,-0.875486806182028,1.4060340511004,-1.52569833771551 +94,1,0,-0.257821470178811,0.103041140124536,3.00578229260879 +94,2,0,-0.257821470178811,0.0510455966337183,2.81600739821049 +94,3,0,-0.257821470178811,1.28170651334834,1.88866826165356 +94,4,0,-0.257821470178811,0.524450670730671,4.24222926203413 +94,5,0,-0.257821470178811,0.543757953082137,3.89014943444177 +94,6,0,-0.257821470178811,0.837694358790485,5.33137502521585 +95,1,0,-1.06522102729796,0.338860131607811,1.32692462722341 +95,2,0,-1.06522102729796,-0.127369030427613,-0.0832637940592554 +95,3,0,-1.06522102729796,0.0207887615637769,-0.11264669981747 +95,4,0,-1.06522102729796,0.226974834769246,2.89905849310711 +95,5,0,-1.06522102729796,0.495476577660107,2.54073657860863 +95,6,0,-1.06522102729796,0.231418798298941,0.847408625748489 +96,1,0,0.55656405447025,1.84356052443496,2.97044415315513 +96,2,0,0.55656405447025,0.403515009694641,4.91746159042525 +96,3,0,0.55656405447025,1.18467123093859,3.82888905681828 +96,4,0,0.55656405447025,1.45829670995655,4.519569208739 +96,5,0,0.55656405447025,2.31876334410251,2.96806867089888 +96,6,0,0.55656405447025,1.45175015135933,4.95375559492303 +97,1,0,0.968155189610845,0.731474808343252,2.00819632359671 +97,2,0,0.968155189610845,1.81966272467269,2.41871187476564 +97,3,0,0.968155189610845,1.32541288635434,2.85560582347779 +97,4,0,0.968155189610845,1.02840238957306,1.99543376514698 +97,5,0,0.968155189610845,0.72402655805433,6.36901987614095 +97,6,0,0.968155189610845,1.62812732719515,5.45740806337183 +98,1,0,-0.642161084239234,-0.285123013459829,-0.000642853350642403 +98,2,0,-0.642161084239234,-0.598332460697446,2.16895969756833 +98,3,0,-0.642161084239234,0.455815886578303,0.909661380366459 +98,4,0,-0.642161084239234,0.799759144823868,2.97501355379955 +98,5,0,-0.642161084239234,0.742981106632073,2.0103010060523 +98,6,0,-0.642161084239234,1.28403169128433,3.27781589866413 +99,1,0,0.489663886549812,1.58049905561598,0.63224613149953 +99,2,0,0.489663886549812,0.372598634840632,0.153880335628493 +99,3,0,0.489663886549812,1.4943622173095,1.81815428213934 +99,4,0,0.489663886549812,1.5378597808116,0.935298801828264 +99,5,0,0.489663886549812,1.56518687462083,2.92415904318258 +99,6,0,0.489663886549812,0.602576741552375,2.44369000347583 +100,1,0,0.83152065605674,0.478913216287586,0.746784055135776 +100,2,0,0.83152065605674,1.4235529798745,2.04540124845096 +100,3,0,0.83152065605674,0.968193684526933,1.32348962919775 +100,4,0,0.83152065605674,1.07833225792295,0.547613792696509 +100,5,0,0.83152065605674,1.06746551895251,3.31761291520659 +100,6,0,0.83152065605674,1.85523532671524,3.08316773932352 +101,1,0,-0.919726134671162,-0.0153717261852472,2.7788719289477 +101,2,0,-0.919726134671162,0.0258159670995393,2.62781854772703 +101,3,0,-0.919726134671162,-0.210141999024792,2.44073176765192 +101,4,0,-0.919726134671162,0.524799330002078,1.78160512517281 +101,5,0,-0.919726134671162,-0.0437030303186893,3.04165380941417 +101,6,0,-0.919726134671162,1.3026199543081,5.32428649347593 +102,1,0,0.641255824240951,0.532093223730807,-2.09251354544218 +102,2,0,0.641255824240951,0.0941130407259139,0.733006859014464 +102,3,0,0.641255824240951,1.55861283645998,-0.23990034173479 +102,4,0,0.641255824240951,1.21632707133294,-0.176646402386964 +102,5,0,0.641255824240951,1.77130479077965,4.52178543711811 +102,6,0,0.641255824240951,2.00779025241614,1.91109596784127 +103,1,0,1.14326851073882,0.0119837643154777,2.22043326561435 +103,2,0,1.14326851073882,0.362873618700088,4.6975800148237 +103,3,0,1.14326851073882,1.97187455363987,2.20278367481176 +103,4,0,1.14326851073882,0.549403280336012,4.53412190050463 +103,5,0,1.14326851073882,0.74357105113021,4.04691725702911 +103,6,0,1.14326851073882,1.41192305654462,5.38945740422014 +104,1,0,-1.08072100327974,-0.36568086508946,-0.464035936891244 +104,2,0,-1.08072100327974,-0.98836432896696,-0.356166326023515 +104,3,0,-1.08072100327974,0.467378110330098,-0.0636497459678805 +104,4,0,-1.08072100327974,-0.935126213484022,-0.716266458809519 +104,5,0,-1.08072100327974,0.502717496775458,-0.660129046615434 +104,6,0,-1.08072100327974,1.47173852447056,2.01181621362982 +105,1,0,-0.308977011536411,-0.179345642251318,-0.562705733283704 +105,2,0,-0.308977011536411,0.70656302910021,0.18167481062825 +105,3,0,-0.308977011536411,1.21257374180067,0.813478648356054 +105,4,0,-0.308977011536411,0.381854939551222,0.207736765764304 +105,5,0,-0.308977011536411,1.2106178506653,0.0126906713691238 +105,6,0,-0.308977011536411,0.964448644390534,1.03945464476557 +106,1,0,1.50226005393992,1.34925401718096,2.64319435904704 +106,2,0,1.50226005393992,0.98662538563711,1.3596364327739 +106,3,0,1.50226005393992,2.12823788384978,3.59777809891119 +106,4,0,1.50226005393992,2.30362907733559,5.55472033802837 +106,5,0,1.50226005393992,1.37909145501961,4.84064460409266 +106,6,0,1.50226005393992,1.71825363617274,4.87571321763657 +107,1,0,-0.89457220946846,-0.463955575441369,-3.48745734772936 +107,2,0,-0.89457220946846,-0.0519919382569897,0.125749218439857 +107,3,0,-0.89457220946846,0.520988865165158,-1.70695509814757 +107,4,0,-0.89457220946846,0.33236786268323,0.725662474427429 +107,5,0,-0.89457220946846,1.03760437559963,1.39038624668232 +107,6,0,-0.89457220946846,0.256795459568391,-0.335002534800443 +108,1,0,-1.22324299509621,-0.526114758338154,-0.692133068311805 +108,2,0,-1.22324299509621,0.18477604071721,0.429974089819244 +108,3,0,-1.22324299509621,0.286754140761167,0.372277811842254 +108,4,0,-1.22324299509621,0.542204942504875,0.697878652044274 +108,5,0,-1.22324299509621,0.399442852524572,1.14482729349123 +108,6,0,-1.22324299509621,1.27799739480744,0.760227606864043 +109,1,0,-0.758669466253882,-0.783003297910145,0.0513878206820995 +109,2,0,-0.758669466253882,0.285020986239263,-0.434316127626099 +109,3,0,-0.758669466253882,0.898896938193642,-1.10042010645776 +109,4,0,-0.758669466253882,0.828259841590877,0.643677661471694 +109,5,0,-0.758669466253882,0.380786238406674,1.65897088961278 +109,6,0,-0.758669466253882,0.456410360436729,1.16614884094835 +110,1,0,-2.28836186963232,-1.42240905596012,1.56363050881366 +110,2,0,-2.28836186963232,-1.21031092491361,2.01509164036705 +110,3,0,-2.28836186963232,-0.766586478955116,0.996018770236634 +110,4,0,-2.28836186963232,-0.412915805901137,1.21019829707552 +110,5,0,-2.28836186963232,-1.10526766855183,1.11050770085309 +110,6,0,-2.28836186963232,-0.0558900611664474,0.75444310848228 +111,1,0,0.79771570665335,-0.553133989340675,1.36541391115453 +111,2,0,0.79771570665335,0.246665282521328,2.08614423172338 +111,3,0,0.79771570665335,0.972160689944622,2.5652568519032 +111,4,0,0.79771570665335,0.376996285841898,2.30036239478882 +111,5,0,0.79771570665335,1.4268445074441,3.26085042603327 +111,6,0,0.79771570665335,1.94231661484459,5.31180379360411 +112,1,0,0.12870078897994,0.307506803447317,0.692955762744788 +112,2,0,0.12870078897994,-0.0268741505941875,2.4379863824427 +112,3,0,0.12870078897994,1.00343228065177,1.30208928752089 +112,4,0,0.12870078897994,0.131765893134764,2.86354301083545 +112,5,0,0.12870078897994,0.933523097093359,2.94208138077174 +112,6,0,0.12870078897994,1.76925438035231,3.72716043227665 +113,1,0,-0.00931074971757802,-0.397545128543512,-0.304337726872656 +113,2,0,-0.00931074971757802,0.504335269263596,-1.47696043309091 +113,3,0,-0.00931074971757802,0.69556377664362,0.706831865285902 +113,4,0,-0.00931074971757802,0.952311641049705,1.58895831015197 +113,5,0,-0.00931074971757802,1.72201418175824,1.80348015736981 +113,6,0,-0.00931074971757802,1.00346429177833,2.07938285516792 +114,1,0,-1.33814559514493,-0.13084645095529,0.258563622937842 +114,2,0,-1.33814559514493,-0.328313780755942,-0.755357255404722 +114,3,0,-1.33814559514493,0.767190991540338,0.547314979192388 +114,4,0,-1.33814559514493,0.551545429130956,1.51813317281849 +114,5,0,-1.33814559514493,0.280707403536271,0.585742471844865 +114,6,0,-1.33814559514493,1.87073018189421,1.79129333628737 +115,1,0,-0.0307450017978104,-0.203832382523738,-0.547529905930902 +115,2,0,-0.0307450017978104,0.803531851748503,-1.94762176066094 +115,3,0,-0.0307450017978104,-0.328550830765483,1.15802858200512 +115,4,0,-0.0307450017978104,1.08894515645791,0.479216783688697 +115,5,0,-0.0307450017978104,1.00890444877836,2.146119939363 +115,6,0,-0.0307450017978104,2.54984322597835,2.60041936200757 +116,1,0,0.416210502177554,0.800227902503501,-1.13776949509889 +116,2,0,0.416210502177554,0.72177376351145,-1.29095759397637 +116,3,0,0.416210502177554,1.05092852687046,0.163616534216729 +116,4,0,0.416210502177554,1.07841177778874,0.318569963738997 +116,5,0,0.416210502177554,1.27216409348855,2.62571443309508 +116,6,0,0.416210502177554,2.20291413903546,1.40992567038327 +117,1,0,0.185682053987045,0.0154632265573412,0.126843635097071 +117,2,0,0.185682053987045,-0.294568652078405,-1.59502600932928 +117,3,0,0.185682053987045,0.3434221572718,-0.0954442306696356 +117,4,0,0.185682053987045,0.736971931659641,0.410252145148292 +117,5,0,0.185682053987045,1.5296434146358,1.91722421838383 +117,6,0,0.185682053987045,1.67782346096464,1.01747266376817 +118,1,0,-1.59532950598604,-0.798373104920996,-1.00968400645432 +118,2,0,-1.59532950598604,-0.204361624072532,0.370830252251386 +118,3,0,-1.59532950598604,-0.896149987574084,0.484139473997054 +118,4,0,-1.59532950598604,0.141700472907281,-0.466405186376612 +118,5,0,-1.59532950598604,1.00839653260119,-0.026721952059122 +118,6,0,-1.59532950598604,0.332064509443496,0.821121599465879 +119,1,0,0.121268709978484,0.181681164145453,0.570191327984123 +119,2,0,0.121268709978484,0.519288120645323,2.03742277716536 +119,3,0,0.121268709978484,0.742228904360963,0.281887395792038 +119,4,0,0.121268709978484,0.580678820252124,2.26926858657859 +119,5,0,0.121268709978484,0.646598927456894,0.783493583393684 +119,6,0,0.121268709978484,1.40112832662989,0.82104317069335 +120,1,0,-0.523538252561795,-0.179295568123839,0.22212232449333 +120,2,0,-0.523538252561795,0.10946622333482,0.335398721679965 +120,3,0,-0.523538252561795,0.89486358654903,0.578696943291453 +120,4,0,-0.523538252561795,0.176361255514506,1.50645843591311 +120,5,0,-0.523538252561795,0.706148111155747,4.27936001732894 +120,6,0,-0.523538252561795,1.38773488273727,3.45033864189556 +121,1,3,-1.3209898980621,-0.659030256624808,-1.11457919929402 +121,2,3,-1.3209898980621,-0.141217731860885,1.28435903211288 +121,3,3,-1.3209898980621,-0.791920375267903,0.685288874745403 +121,4,3,-1.3209898980621,-0.0734820400267869,3.25785366872356 +121,5,3,-1.3209898980621,-0.49656205388661,2.02276889139347 +121,6,3,-1.3209898980621,-0.965589275967642,5.21429798519067 +122,1,3,0.65557245598294,0.507227828874015,-0.250570912893117 +122,2,3,0.65557245598294,1.03929914437131,-0.420264983423434 +122,3,3,0.65557245598294,1.34473746170499,1.76685337912262 +122,4,3,0.65557245598294,0.87359591944603,3.4820602234097 +122,5,3,0.65557245598294,0.892485396451848,5.95175331176178 +122,6,3,0.65557245598294,2.52111655846667,7.40612320147572 +123,1,3,0.0144327369651312,0.897110333973617,-0.887945988958214 +123,2,3,0.0144327369651312,0.562597501739654,0.177689505327719 +123,3,3,0.0144327369651312,1.19295988538516,2.78876965030089 +123,4,3,0.0144327369651312,-0.152912404575872,2.14857714673905 +123,5,3,0.0144327369651312,0.354572848563497,5.03305075612627 +123,6,3,0.0144327369651312,0.11038766791842,6.34960584861583 +124,1,3,0.06591379512311,0.0534203651908569,-0.843874058964572 +124,2,3,0.06591379512311,0.270394743140451,2.86834595033898 +124,3,3,0.06591379512311,0.129802985645562,3.91738873706921 +124,4,3,0.06591379512311,0.873770307631855,4.42769196153781 +124,5,3,0.06591379512311,0.983498849130909,7.0964325924724 +124,6,3,0.06591379512311,1.20719003484248,9.48929410346392 +125,1,3,-0.275046326426422,0.293249514156622,1.81714098956309 +125,2,3,-0.275046326426422,0.229476930121458,0.185804954844934 +125,3,3,-0.275046326426422,0.268736748455175,2.76347481395949 +125,4,3,-0.275046326426422,0.466683387349963,4.5530580706989 +125,5,3,-0.275046326426422,1.15612694018968,7.60432928703051 +125,6,3,-0.275046326426422,0.396102765747029,5.50714869110561 +126,1,3,-0.556089165281445,-0.204073561439021,0.917827508857894 +126,2,3,-0.556089165281445,0.19698200850372,2.39608998202929 +126,3,3,-0.556089165281445,1.33309830602468,2.51649382846799 +126,4,3,-0.556089165281445,0.813016580393121,3.75983978920604 +126,5,3,-0.556089165281445,0.279270401241487,5.81598295015132 +126,6,3,-0.556089165281445,0.767015417251097,6.57061869757607 +127,1,3,-0.481761250465059,-0.507168035910364,-0.907680621931876 +127,2,3,-0.481761250465059,0.782178870451505,-1.068489152912 +127,3,3,-0.481761250465059,-0.112056540437024,1.85024307065664 +127,4,3,-0.481761250465059,0.398656788397188,5.22249311314911 +127,5,3,-0.481761250465059,0.423006794578998,3.11951784142225 +127,6,3,-0.481761250465059,0.671986212079901,5.44040257022184 +128,1,3,-1.32740904200675,-1.65144181693399,1.33668575620608 +128,2,3,-1.32740904200675,0.13845682943084,-0.164424994709493 +128,3,3,-1.32740904200675,-0.186641909443694,1.74404216922443 +128,4,3,-1.32740904200675,-0.0193098809394121,1.16926685493817 +128,5,3,-1.32740904200675,0.260278867551969,3.41050128420705 +128,6,3,-1.32740904200675,0.426968699743162,5.81478718328758 +129,1,3,-0.835635156209953,0.365077037473928,1.17563749280077 +129,2,3,-0.835635156209953,0.0255047424585454,0.463487723132613 +129,3,3,-0.835635156209953,0.238261763497732,1.81748991235753 +129,4,3,-0.835635156209953,1.14995136482981,3.02643141094252 +129,5,3,-0.835635156209953,-0.368831270681908,4.53077069457151 +129,6,3,-0.835635156209953,0.494855790328605,3.7369366925089 +130,1,3,0.014617056865741,-0.920193437062664,-0.468859478425729 +130,2,3,0.014617056865741,-0.409771247075564,-0.195779473995372 +130,3,3,0.014617056865741,0.143316992860131,3.35793046974238 +130,4,3,0.014617056865741,1.16344796814541,2.90092316112646 +130,5,3,0.014617056865741,1.92474818134048,7.05539446585689 +130,6,3,0.014617056865741,1.23561893888835,7.4371222065699 +131,1,3,0.438622701805894,1.00256898424452,0.535588551592362 +131,2,3,0.438622701805894,0.891298196316596,-0.51181417381938 +131,3,3,0.438622701805894,1.09060025694346,2.97164644917054 +131,4,3,0.438622701805894,1.34453810826578,2.25079123994776 +131,5,3,0.438622701805894,1.40699884669438,3.59911808574875 +131,6,3,0.438622701805894,0.939875156111874,7.29199956126919 +132,1,3,-1.76243810418353,-1.55642713172473,-0.805043158381241 +132,2,3,-1.76243810418353,-0.391682064940616,-2.0263867595638 +132,3,3,-1.76243810418353,-0.666139512388104,-1.05765094965184 +132,4,3,-1.76243810418353,0.0238185660916042,0.817677818881005 +132,5,3,-1.76243810418353,0.400946115070112,2.91380467653736 +132,6,3,-1.76243810418353,0.913005144984613,2.69049089180893 +133,1,3,1.36534824167319,0.763831528913434,0.577023159081942 +133,2,3,1.36534824167319,0.844309096416079,0.042038282212332 +133,3,3,1.36534824167319,0.947415268176042,2.18378635404389 +133,4,3,1.36534824167319,1.43995238461224,4.92965591378078 +133,5,3,1.36534824167319,2.89808347087941,8.12380754151815 +133,6,3,1.36534824167319,1.10219087008148,9.40004516409809 +134,1,3,1.29446825743501,0.608144869107727,1.08618105043624 +134,2,3,1.29446825743501,1.41973826830158,2.32331581819127 +134,3,3,1.29446825743501,1.51878532055069,3.65140165693938 +134,4,3,1.29446825743501,1.69646592321119,5.36297031018112 +134,5,3,1.29446825743501,1.56911161772488,8.48595978983258 +134,6,3,1.29446825743501,1.90002513778021,9.35068411797129 +135,1,3,-0.178549336156913,0.151158553945093,0.828144325264999 +135,2,3,-0.178549336156913,-0.471060546327792,3.9089119030933 +135,3,3,-0.178549336156913,0.103993512674667,6.07766753358876 +135,4,3,-0.178549336156913,1.08664644183412,5.71413666854425 +135,5,3,-0.178549336156913,1.68404926041005,5.99317910927949 +135,6,3,-0.178549336156913,0.577078077823259,9.483960218053 +136,1,3,-1.07692090882478,-0.401929333738954,1.04747553448351 +136,2,3,-1.07692090882478,-0.545038593804227,0.890555345254023 +136,3,3,-1.07692090882478,0.508562503370516,0.983523883778991 +136,4,3,-1.07692090882478,0.00140170282829649,2.45797787912278 +136,5,3,-1.07692090882478,0.13965802945591,4.85004362804588 +136,6,3,-1.07692090882478,0.935084069218439,5.46362892185911 +137,1,3,0.728454193443001,-0.0114977260864277,0.974364596857209 +137,2,3,0.728454193443001,0.7135343982297,0.67155879830354 +137,3,3,0.728454193443001,0.655179531278547,2.90146090914518 +137,4,3,0.728454193443001,1.38622403146752,5.33426163164905 +137,5,3,0.728454193443001,1.12156269309693,7.6727252728244 +137,6,3,0.728454193443001,1.15693441030068,8.86462680988762 +138,1,3,-1.46938141572406,-0.0883249129282256,0.722072099170389 +138,2,3,-1.46938141572406,-0.142449306634193,0.823877896167457 +138,3,3,-1.46938141572406,-0.0298040848424808,2.30322550088994 +138,4,3,-1.46938141572406,-0.322736256121953,2.31264370539726 +138,5,3,-1.46938141572406,0.124282773210025,6.14394845899594 +138,6,3,-1.46938141572406,1.1023978140493,5.08916101818471 +139,1,3,-0.549298690203426,0.301171733824676,-0.227928347441122 +139,2,3,-0.549298690203426,0.402237345797786,-0.737524166727403 +139,3,3,-0.549298690203426,0.683518954814073,-1.0638203854028 +139,4,3,-0.549298690203426,0.403594704560704,2.16485431597404 +139,5,3,-0.549298690203426,0.55999493763121,4.51625222775326 +139,6,3,-0.549298690203426,1.06915987676031,4.39776263951282 +140,1,3,-1.07513120640263,-0.218917779907026,0.741651877086614 +140,2,3,-1.07513120640263,-0.412691091490353,-0.171855503087313 +140,3,3,-1.07513120640263,0.367202949190019,0.58558380183808 +140,4,3,-1.07513120640263,1.24324583274143,2.70757401857652 +140,5,3,-1.07513120640263,0.424084621174276,1.96616566887664 +140,6,3,-1.07513120640263,0.534559115396839,4.59790385921341 +141,1,3,0.589903379971464,0.518152792811317,0.532857161426118 +141,2,3,0.589903379971464,1.74119554926273,0.583690811233439 +141,3,3,0.589903379971464,1.14384744906574,1.11047142482919 +141,4,3,0.589903379971464,1.43423263831571,5.00913061015248 +141,5,3,0.589903379971464,1.74498549823101,6.56368416729218 +141,6,3,0.589903379971464,1.67289590250084,9.02370809506211 +142,1,3,0.657929099074857,-0.527630501829893,2.23509371293706 +142,2,3,0.657929099074857,0.790313265176156,3.28376796139343 +142,3,3,0.657929099074857,1.30011710750872,1.89310004655243 +142,4,3,0.657929099074857,1.27724881504038,5.63969392962316 +142,5,3,0.657929099074857,0.767272841138719,6.94525677505373 +142,6,3,0.657929099074857,2.32556023319219,9.74629505340234 +143,1,3,0.918209578132046,0.389990741964451,1.73002264959394 +143,2,3,0.918209578132046,0.324124967116553,2.51685777627825 +143,3,3,0.918209578132046,0.864710010354607,4.10998227389613 +143,4,3,0.918209578132046,0.721106454137181,5.74373385047243 +143,5,3,0.918209578132046,2.35220290569798,6.31425548414344 +143,6,3,0.918209578132046,1.06086818047735,9.11988048390896 +144,1,3,1.03723233079857,0.780710632652874,0.212208576948754 +144,2,3,1.03723233079857,1.05604424585774,-0.667835264144556 +144,3,3,1.03723233079857,0.452858041598415,2.84834674243207 +144,4,3,1.03723233079857,1.974918457812,5.49179716277523 +144,5,3,1.03723233079857,1.04103576092729,6.15510734290007 +144,6,3,1.03723233079857,2.13124097931571,10.0580109007073 +145,1,3,0.432864420882128,0.0546511643881509,2.06577211453651 +145,2,3,0.432864420882128,0.542564453524272,0.826589014872826 +145,3,3,0.432864420882128,-0.102986095234928,3.30224330441748 +145,4,3,0.432864420882128,0.818817860108457,5.56089893194988 +145,5,3,0.432864420882128,0.961458251664878,7.7240973086671 +145,6,3,0.432864420882128,1.27895634770374,8.02085741371424 +146,1,3,0.494887951274981,0.906621933718628,1.35833730554146 +146,2,3,0.494887951274981,0.609584098447873,1.50672641372537 +146,3,3,0.494887951274981,1.66806988803927,2.68420111093815 +146,4,3,0.494887951274981,1.17243386474274,5.77226847745143 +146,5,3,0.494887951274981,1.7367792550661,6.86343483473947 +146,6,3,0.494887951274981,1.32962006663717,7.39856300627146 +147,1,3,0.337514991797093,0.714474564182813,3.77502417711785 +147,2,3,0.337514991797093,-0.120717345754449,2.39855290111283 +147,3,3,0.337514991797093,0.979181408407107,3.3951013798209 +147,4,3,0.337514991797093,0.765822372463936,4.51187239398326 +147,5,3,0.337514991797093,1.1636592509228,7.32871256559361 +147,6,3,0.337514991797093,1.69073725736773,7.89889121515618 +148,1,3,0.940914420491235,0.32599988959407,-2.68432419188846 +148,2,3,0.940914420491235,0.902667702594575,-0.502223665400725 +148,3,3,0.940914420491235,1.79711547389592,2.72616983157362 +148,4,3,0.940914420491235,1.89564536968462,3.39439552886371 +148,5,3,0.940914420491235,1.86546638275212,4.46411002977733 +148,6,3,0.940914420491235,1.85126607219131,5.8921851033331 +149,1,3,0.457967792328404,0.378698935479082,4.12572103429067 +149,2,3,0.457967792328404,0.625145122806749,4.12170173700891 +149,3,3,0.457967792328404,0.657407339867929,5.67039906683291 +149,4,3,0.457967792328404,0.894496831842379,8.9524316705688 +149,5,3,0.457967792328404,1.27815558368229,9.34940165660425 +149,6,3,0.457967792328404,1.21757039960575,9.72894783793935 +150,1,3,1.28822951042595,0.929280238078882,-0.681541858459146 +150,2,3,1.28822951042595,0.37556044800664,-2.14029291717815 +150,3,3,1.28822951042595,0.955960970286821,1.49994481205458 +150,4,3,1.28822951042595,0.666797039509379,3.18273482219049 +150,5,3,1.28822951042595,1.93863479694882,6.10370652383497 +150,6,3,1.28822951042595,1.3261231323096,5.09851390581681 +151,1,3,0.799327933544311,0.752383684129809,0.998835172417796 +151,2,3,0.799327933544311,1.48794166496513,2.50739666840068 +151,3,3,0.799327933544311,0.636647413978102,3.39248141555062 +151,4,3,0.799327933544311,0.43183050965539,6.2241784706405 +151,5,3,0.799327933544311,1.60186143581385,6.57526813277581 +151,6,3,0.799327933544311,1.48009902563213,9.68047322868392 +152,1,3,0.641075436191527,0.410381761209829,0.181288378872355 +152,2,3,0.641075436191527,0.833387068122976,0.440335510368963 +152,3,3,0.641075436191527,1.02324616435653,1.92203052266619 +152,4,3,0.641075436191527,1.77222798638308,3.95065028029737 +152,5,3,0.641075436191527,1.50187251866824,2.58165781357217 +152,6,3,0.641075436191527,0.913951107889827,6.90369945886804 +153,1,3,0.647106275420512,-0.237400237086652,-2.02164808965544 +153,2,3,0.647106275420512,0.257978734055656,0.228139264140656 +153,3,3,0.647106275420512,1.75480644001202,-0.169015192170175 +153,4,3,0.647106275420512,0.41350545765392,3.79498507213573 +153,5,3,0.647106275420512,1.07445417959469,4.38245420741927 +153,6,3,0.647106275420512,0.98341038645438,6.61511100339346 +154,1,3,1.04666451517136,0.341354827336876,3.42913227448808 +154,2,3,1.04666451517136,0.672471846348521,0.616635516261048 +154,3,3,1.04666451517136,1.27811698735899,2.7748662991019 +154,4,3,1.04666451517136,1.48649127087623,6.01209090483621 +154,5,3,1.04666451517136,1.61543543912908,7.37520919143932 +154,6,3,1.04666451517136,2.02536607455159,9.48209414318584 +155,1,3,0.576975845608666,0.0914320774480271,0.722851227802534 +155,2,3,0.576975845608666,0.108522327794117,1.6890512266516 +155,3,3,0.576975845608666,0.259628061168946,2.58597206970974 +155,4,3,0.576975845608666,1.01811928925711,6.21498745277975 +155,5,3,0.576975845608666,0.719060589759526,7.07453482145899 +155,6,3,0.576975845608666,1.92334185803099,10.009696002852 +156,1,3,0.17037629608571,0.784349410920146,2.11235841463052 +156,2,3,0.17037629608571,0.214232082289887,-0.697328657084354 +156,3,3,0.17037629608571,1.25013551445887,3.29808927709344 +156,4,3,0.17037629608571,1.7086115578765,4.77325299010961 +156,5,3,0.17037629608571,1.11415545041536,5.70327263182313 +156,6,3,0.17037629608571,0.847156272453002,6.739097993206 +157,1,3,0.396115427083706,0.923645518828156,0.864477515155358 +157,2,3,0.396115427083706,0.648033309117745,3.31731508957503 +157,3,3,0.396115427083706,1.14817613766954,3.60719373954189 +157,4,3,0.396115427083706,0.420299068553708,5.09071471525354 +157,5,3,0.396115427083706,1.14833117298122,6.75113263123557 +157,6,3,0.396115427083706,1.07770858221266,8.92370742505083 +158,1,3,-0.38503981699399,0.249091044450303,-0.455481254742036 +158,2,3,-0.38503981699399,0.652965111326567,0.513194049834224 +158,3,3,-0.38503981699399,-0.203526544921249,3.97850435486181 +158,4,3,-0.38503981699399,0.178102153152252,3.50022698865287 +158,5,3,-0.38503981699399,0.238824839896899,5.90492745057116 +158,6,3,-0.38503981699399,1.41010656792478,6.139197871099 +159,1,3,-0.606966899370249,-0.235070624391252,1.67459208020634 +159,2,3,-0.606966899370249,0.397058569815924,-0.983876672431654 +159,3,3,-0.606966899370249,-0.0987236791816803,2.11597559238572 +159,4,3,-0.606966899370249,0.0175133414764335,4.93858440091871 +159,5,3,-0.606966899370249,1.07180450901133,6.65025467972839 +159,6,3,-0.606966899370249,1.03878178919866,4.64039122565767 +160,1,3,0.183609061947842,0.922644868366452,1.06263247541716 +160,2,3,0.183609061947842,-0.521822880184819,3.84919810086592 +160,3,3,0.183609061947842,1.26182116417092,2.87398711588164 +160,4,3,0.183609061947842,1.1296518783099,4.97402442219597 +160,5,3,0.183609061947842,1.16806292711789,6.59628616328911 +160,6,3,0.183609061947842,1.54910874984912,7.78160398847949 +161,1,3,-1.5739802437226,-0.738750875893334,0.646798997035059 +161,2,3,-1.5739802437226,-0.817190355326895,-0.199794544446061 +161,3,3,-1.5739802437226,0.00978813383769062,1.53768978518043 +161,4,3,-1.5739802437226,-0.0692027187562883,4.22559029952303 +161,5,3,-1.5739802437226,-0.321648430686517,2.49489898430561 +161,6,3,-1.5739802437226,1.80291081637126,2.97534946673223 +162,1,3,-1.5135182536791,-1.21415498016749,1.08344375432323 +162,2,3,-1.5135182536791,0.00862446836171632,0.366096436545889 +162,3,3,-1.5135182536791,-0.580560672020834,0.386396282908581 +162,4,3,-1.5135182536791,0.284802496374087,2.65431055942517 +162,5,3,-1.5135182536791,0.481985729465867,4.08004569199787 +162,6,3,-1.5135182536791,-0.164457779819998,5.44486509744182 +163,1,3,0.17709443103689,0.282609663048505,1.2187601244373 +163,2,3,0.17709443103689,0.411582509889598,3.46647644603637 +163,3,3,0.17709443103689,0.847495649227131,6.24284448978847 +163,4,3,0.17709443103689,1.00780264364244,6.26054223704011 +163,5,3,0.17709443103689,0.567458999918712,6.75896719170673 +163,6,3,0.17709443103689,1.73124741621038,8.55153680922603 +164,1,3,0.403250756295859,0.492850083886365,2.48897701813397 +164,2,3,0.403250756295859,0.540083950359147,2.10861656799587 +164,3,3,0.403250756295859,0.731550492755462,5.78483021182663 +164,4,3,0.403250756295859,0.458474142754207,6.25304959523288 +164,5,3,0.403250756295859,1.4995702617515,7.24373626356283 +164,6,3,0.403250756295859,0.984707497276815,8.31693696597542 +165,1,3,-1.75890966008881,-0.492384152667725,-3.0165269004572 +165,2,3,-1.75890966008881,-0.114806693432703,1.51062699564072 +165,3,3,-1.75890966008881,-0.48595164161601,-1.00828460304626 +165,4,3,-1.75890966008881,0.617535901530216,3.78881684855537 +165,5,3,-1.75890966008881,-0.477578070519953,4.53617118871654 +165,6,3,-1.75890966008881,0.0165545486318738,2.41304871152758 +166,1,3,0.550444344652174,0.720896381917308,-0.0902337840097769 +166,2,3,0.550444344652174,1.07643595531667,1.67174317972133 +166,3,3,0.550444344652174,1.23330497683997,3.30095700316899 +166,4,3,0.550444344652174,0.239620344875785,5.01348139892779 +166,5,3,0.550444344652174,0.572685885770811,5.04458155047439 +166,6,3,0.550444344652174,1.90034510518233,8.66633628083652 +167,1,3,-0.177518472281448,0.559127011742701,-2.84585414560063 +167,2,3,-0.177518472281448,0.671231625413232,0.97648769056059 +167,3,3,-0.177518472281448,0.596607991549095,0.67623314360395 +167,4,3,-0.177518472281448,0.28242453789717,2.26063148836001 +167,5,3,-0.177518472281448,0.772759320165847,4.26137543656067 +167,6,3,-0.177518472281448,0.0529782047803846,4.03342527507047 +168,1,3,-0.283378689715887,0.56880477901632,0.566393501073422 +168,2,3,-0.283378689715887,0.687228894918578,1.33767774653906 +168,3,3,-0.283378689715887,0.716981777257136,3.20528354672777 +168,4,3,-0.283378689715887,0.589459989413421,4.6418581200693 +168,5,3,-0.283378689715887,0.896581550535394,5.35234524586164 +168,6,3,-0.283378689715887,1.16641332038933,5.89859615929218 +169,1,3,0.0949255069380952,-0.138844826813786,0.880456211503483 +169,2,3,0.0949255069380952,0.899054228996397,1.70093245545876 +169,3,3,0.0949255069380952,0.375520297802488,2.34651154150143 +169,4,3,0.0949255069380952,0.765847242429938,7.0065917773773 +169,5,3,0.0949255069380952,0.770509647956809,4.98968023938993 +169,6,3,0.0949255069380952,1.34483793919838,6.9101710957784 +170,1,3,0.109708934474219,0.386120730420973,2.85160717995104 +170,2,3,0.109708934474219,0.407879747082585,1.33752020856814 +170,3,3,0.109708934474219,0.637178862432442,4.20098703126136 +170,4,3,0.109708934474219,0.207565281405001,4.77374671451531 +170,5,3,0.109708934474219,1.85419902965847,6.3939200558063 +170,6,3,0.109708934474219,1.61088867029117,6.59919416619873 +171,1,3,-1.02075201769416,-0.356310807707324,1.02592651163283 +171,2,3,-1.02075201769416,0.0424193927151826,0.741657135044775 +171,3,3,-1.02075201769416,0.0317264399471442,1.71370818238931 +171,4,3,-1.02075201769416,0.467235778780831,5.47811263281214 +171,5,3,-1.02075201769416,0.0791128775261391,4.12329110395588 +171,6,3,-1.02075201769416,0.895303915215752,5.00292845449822 +172,1,3,-0.214471204357173,-0.727646095799363,1.35131925370431 +172,2,3,-0.214471204357173,0.210944497892413,2.37175026318938 +172,3,3,-0.214471204357173,0.570841146082983,3.65951143912713 +172,4,3,-0.214471204357173,0.461650424820328,6.3022531708993 +172,5,3,-0.214471204357173,0.342228491003671,8.22419778472385 +172,6,3,-0.214471204357173,1.28117396857179,7.84810036822843 +173,1,3,0.95312723300276,0.587136635676131,2.27117731500969 +173,2,3,0.95312723300276,0.467985411032795,2.69182871995691 +173,3,3,0.95312723300276,0.768091000867892,6.44965476196657 +173,4,3,0.95312723300276,1.53518612893103,6.20379049823067 +173,5,3,0.95312723300276,0.734032956366248,8.37840205107447 +173,6,3,0.95312723300276,0.880826456225791,10.6071742329299 +174,1,3,-0.299503059426975,-0.0143918053073009,0.165378654819905 +174,2,3,-0.299503059426975,0.351702004390223,1.14652818851001 +174,3,3,-0.299503059426975,1.35443661771852,1.34930878825417 +174,4,3,-0.299503059426975,0.641451213769214,2.97573642323419 +174,5,3,-0.299503059426975,0.792151951680742,3.2784114112508 +174,6,3,-0.299503059426975,0.559389848134574,7.10103746423127 +175,1,3,-1.35656841055337,-0.68822256639622,1.09383253065236 +175,2,3,-1.35656841055337,0.165950349733725,2.32705481637459 +175,3,3,-1.35656841055337,-0.0394690022428881,1.58889766941899 +175,4,3,-1.35656841055337,-0.304479039922878,0.857436014714444 +175,5,3,-1.35656841055337,1.07142210914254,2.26750720166773 +175,6,3,-1.35656841055337,1.22342980686212,3.62877035996801 +176,1,3,-0.722906834501834,0.163769122541646,-0.632232933093176 +176,2,3,-0.722906834501834,0.0157880302844722,0.938801879860888 +176,3,3,-0.722906834501834,0.317548010285299,3.67834365574177 +176,4,3,-0.722906834501834,0.320655034651504,2.99198807090912 +176,5,3,-0.722906834501834,1.20799091004943,3.5296721551273 +176,6,3,-0.722906834501834,1.4715576177269,3.80599815112298 +177,1,3,-0.873027113678424,-0.134710889146149,1.20554182330847 +177,2,3,-0.873027113678424,-0.343203700912485,1.08472943367274 +177,3,3,-0.873027113678424,0.68383414113027,4.85152313687422 +177,4,3,-0.873027113678424,-0.0307043288901434,4.54226192906532 +177,5,3,-0.873027113678424,0.476565413251091,8.21586862915938 +177,6,3,-0.873027113678424,0.692592828471272,7.78871269859934 +178,1,3,0.04325400319841,0.483812986689953,2.23748077948003 +178,2,3,0.04325400319841,0.124723740750307,0.0257740658283268 +178,3,3,0.04325400319841,-0.0236415679346836,2.07893121569823 +178,4,3,0.04325400319841,0.38645207413081,4.73747920429682 +178,5,3,0.04325400319841,0.911079572477391,6.13334250277362 +178,6,3,0.04325400319841,1.56328403641471,5.10634606436181 +179,1,3,-0.705597138651361,-0.203512657961043,0.0705300276670152 +179,2,3,-0.705597138651361,-0.500641996603977,1.80939716425173 +179,3,3,-0.705597138651361,0.443055984083823,1.22782580215329 +179,4,3,-0.705597138651361,0.785627941108739,3.54952364932024 +179,5,3,-0.705597138651361,0.370592747588609,5.44620669723755 +179,6,3,-0.705597138651361,0.784335634233162,6.91807435557 +180,1,3,2.06398866391933,1.05930399943233,-0.00767017810784587 +180,2,3,2.06398866391933,0.974937384643123,1.3889518162926 +180,3,3,2.06398866391933,1.67488127697509,4.941828354601 +180,4,3,2.06398866391933,1.54166970733265,6.37226663940419 +180,5,3,2.06398866391933,1.55908201581147,7.63019049127558 +180,6,3,2.06398866391933,3.13182760994635,11.0824508424514 +181,1,3,0.360194044770426,0.137219648791359,-0.120984723180791 +181,2,3,0.360194044770426,0.38551064785073,1.30910385832044 +181,3,3,0.360194044770426,0.414240176764444,2.40758725231587 +181,4,3,0.360194044770426,1.03798805335224,4.04969508727083 +181,5,3,0.360194044770426,1.26482943472362,7.05018989645842 +181,6,3,0.360194044770426,0.674817592128119,8.06113032044475 +182,1,3,-1.68147033404543,-0.529992351263284,-0.468300262616531 +182,2,3,-1.68147033404543,-0.361074846598351,-0.82128675411022 +182,3,3,-1.68147033404543,0.193761651635151,2.11230702234137 +182,4,3,-1.68147033404543,0.217264661905855,1.47293064025627 +182,5,3,-1.68147033404543,-0.887821017937981,2.24634296880867 +182,6,3,-1.68147033404543,0.741792955070734,3.71110597293242 +183,1,3,-0.196778992696935,-0.646729030890644,-1.00205754402073 +183,2,3,-0.196778992696935,0.0139864871831406,0.953921068284142 +183,3,3,-0.196778992696935,0.0795937312953826,0.860583458955781 +183,4,3,-0.196778992696935,0.837209243653747,4.51956682189724 +183,5,3,-0.196778992696935,0.601170803412534,4.97723339242763 +183,6,3,-0.196778992696935,1.47422885794644,5.32121969953623 +184,1,3,0.730421144749429,-0.139311224235713,0.171081394747952 +184,2,3,0.730421144749429,1.53996280134507,1.94344486906202 +184,3,3,0.730421144749429,0.782149116082784,3.23383839820865 +184,4,3,0.730421144749429,0.609823463790785,2.62356551558351 +184,5,3,0.730421144749429,1.89101976789857,6.35341741228886 +184,6,3,0.730421144749429,1.72754545713741,7.05230497645335 +185,1,3,-0.0863000134300777,0.953213880450527,0.0349463777858372 +185,2,3,-0.0863000134300777,0.24448596475191,0.745217744798797 +185,3,3,-0.0863000134300777,0.817473988609581,3.84902626581413 +185,4,3,-0.0863000134300777,0.341876847218706,3.75460775000166 +185,5,3,-0.0863000134300777,0.648765871009181,5.82819842016675 +185,6,3,-0.0863000134300777,1.0361709300895,6.16150290917458 +186,1,3,0.506421572168642,0.367175153198149,-0.150215409953131 +186,2,3,0.506421572168642,0.567547043778752,1.16468293926085 +186,3,3,0.506421572168642,0.622465315259915,2.3988406787079 +186,4,3,0.506421572168642,0.394663508169827,6.48536024802799 +186,5,3,0.506421572168642,0.884648088357487,5.87931879381252 +186,6,3,0.506421572168642,1.2150386952591,7.23309433364219 +187,1,3,0.241411363100991,0.926007128421722,0.230728131241308 +187,2,3,0.241411363100991,0.613940209317918,0.0577344374340867 +187,3,3,0.241411363100991,1.98051616149487,2.11188576701728 +187,4,3,0.241411363100991,-0.0482207260230793,1.6364886842494 +187,5,3,0.241411363100991,1.13745180089868,3.05389500156471 +187,6,3,0.241411363100991,1.43270154624331,4.83798493222586 +188,1,3,-0.291937535972332,-0.432249249948712,-3.09229985520743 +188,2,3,-0.291937535972332,0.267479708595359,-1.57889714548199 +188,3,3,-0.291937535972332,0.415797657047746,0.287257962183195 +188,4,3,-0.291937535972332,-0.473326451583243,1.62858283902274 +188,5,3,-0.291937535972332,0.250647366273356,1.21906452860634 +188,6,3,-0.291937535972332,1.39466252891878,4.89754797997687 +189,1,3,-1.42673622400026,0.0874786130907167,-0.492174801618482 +189,2,3,-1.42673622400026,-0.360288142109093,-0.590640207208217 +189,3,3,-1.42673622400026,-0.00295717580758077,0.594372840951305 +189,4,3,-1.42673622400026,0.250174512466053,4.34575202958559 +189,5,3,-1.42673622400026,0.331734399880543,2.33611571216873 +189,6,3,-1.42673622400026,0.496807974220238,4.54737715581265 +190,1,3,-0.2115984836605,0.60223614723348,0.833538926670778 +190,2,3,-0.2115984836605,-0.731387152145866,2.22483251339626 +190,3,3,-0.2115984836605,1.278125494643,2.18564298307356 +190,4,3,-0.2115984836605,0.78864245668234,5.51233312764889 +190,5,3,-0.2115984836605,1.26236600590622,6.24849207856658 +190,6,3,-0.2115984836605,1.05785986019084,8.69015711295789 +191,1,5,0.983592225005866,0.685269042736118,1.38335066496562 +191,2,5,0.983592225005866,0.544502809451078,2.63622648994195 +191,3,5,0.983592225005866,1.27722602650698,1.7975669580347 +191,4,5,0.983592225005866,0.921971281461675,4.00241556657792 +191,5,5,0.983592225005866,1.16159622356324,3.40768352363552 +191,6,5,0.983592225005866,1.87368858096347,6.01936776577239 +192,1,5,-0.283157026746315,0.0104512285659374,1.27997348572735 +192,2,5,-0.283157026746315,-0.0834672788684822,-1.10599779071778 +192,3,5,-0.283157026746315,-0.232655886244044,-0.454134990455851 +192,4,5,-0.283157026746315,0.723728064985206,1.68155470824088 +192,5,5,-0.283157026746315,1.6857236853256,1.87155358218702 +192,6,5,-0.283157026746315,1.44827549669874,5.95064220847003 +193,1,5,0.00523970150119662,0.276436364797033,-1.93364739079216 +193,2,5,0.00523970150119662,-0.286764607187908,-1.76658683663304 +193,3,5,0.00523970150119662,0.830021732791269,-1.31917039846352 +193,4,5,0.00523970150119662,0.0683583315476364,-0.585846233099063 +193,5,5,0.00523970150119662,1.41795184080921,1.07536146125153 +193,6,5,0.00523970150119662,0.820462889803177,4.78603679649615 +194,1,5,0.92545213318825,1.31461163819231,2.92951584215972 +194,2,5,0.92545213318825,1.90310839616577,0.226299788014934 +194,3,5,0.92545213318825,1.87138107688812,3.63924728317667 +194,4,5,0.92545213318825,1.47734438773685,4.09859230112481 +194,5,5,0.92545213318825,1.61762449653127,5.5875297297966 +194,6,5,0.92545213318825,1.53505759071181,5.73026764736178 +195,1,5,-0.635670393667737,0.92906119673608,0.727083910123489 +195,2,5,-0.635670393667737,0.945527892927173,1.89213182433102 +195,3,5,-0.635670393667737,-0.0488719634014442,2.509396606672 +195,4,5,-0.635670393667737,0.589292270566448,1.32663557253803 +195,5,5,-0.635670393667737,0.481316722409301,3.19923450497982 +195,6,5,-0.635670393667737,1.08897508741881,4.12014467427658 +196,1,5,1.15670868958139,0.524265461334005,1.14361307382569 +196,2,5,1.15670868958139,1.09018652226291,1.7777630785721 +196,3,5,1.15670868958139,1.17365751121989,2.80845429477206 +196,4,5,1.15670868958139,0.519026773875079,4.76364255763518 +196,5,5,1.15670868958139,1.52994359394921,4.67118251896483 +196,6,5,1.15670868958139,1.00861805373324,7.97521550687341 +197,1,5,1.78969021425029,0.4434355361539,1.75105876560352 +197,2,5,1.78969021425029,2.26138915469364,3.18412757427007 +197,3,5,1.78969021425029,2.03441329772634,4.87704223939755 +197,4,5,1.78969021425029,2.19733416987764,5.07213816231237 +197,5,5,1.78969021425029,1.86882625281979,7.36445273426155 +197,6,5,1.78969021425029,0.869774157574492,7.89045853098738 +198,1,5,-0.149019232380438,0.906656488536891,0.617582157062771 +198,2,5,-0.149019232380438,-0.303441296388933,1.74706459113415 +198,3,5,-0.149019232380438,-0.0386216673435962,1.41188339797885 +198,4,5,-0.149019232380438,0.227646993112056,2.47893997311591 +198,5,5,-0.149019232380438,1.44183369734344,3.68740351555302 +198,6,5,-0.149019232380438,0.337064359767777,4.93171424134225 +199,1,5,-0.261067529046921,1.10193513549883,4.07397809127892 +199,2,5,-0.261067529046921,0.831821005947784,2.64115271413386 +199,3,5,-0.261067529046921,1.0507459873153,2.5984635365937 +199,4,5,-0.261067529046921,0.954457862577376,3.15005778434285 +199,5,5,-0.261067529046921,0.726355188233059,6.09140004175636 +199,6,5,-0.261067529046921,0.925631026478668,6.26570200917389 +200,1,5,-0.279422558021935,-1.14039912066845,-3.71994209961769 +200,2,5,-0.279422558021935,-1.06989083106309,-1.03919624114515 +200,3,5,-0.279422558021935,1.36983043403129,-0.00638631184268384 +200,4,5,-0.279422558021935,1.08189923332103,0.228665527854604 +200,5,5,-0.279422558021935,0.902665178547284,2.26697554552666 +200,6,5,-0.279422558021935,0.731355545786545,6.36441088594964 +201,1,5,-0.867508001059406,-0.233675811236789,-1.49758007707428 +201,2,5,-0.867508001059406,-0.550883931428525,0.374386856211282 +201,3,5,-0.867508001059406,0.0857728512645791,0.0294364273898407 +201,4,5,-0.867508001059406,-0.232389250145715,1.42191855779375 +201,5,5,-0.867508001059406,1.33590509096511,2.55688670984156 +201,6,5,-0.867508001059406,0.534449929648672,1.42124688873966 +202,1,5,-0.266028106354418,-0.128148387025209,0.975074215504778 +202,2,5,-0.266028106354418,0.446467815751043,-0.0510663436929477 +202,3,5,-0.266028106354418,0.0897105442552565,1.67391335328299 +202,4,5,-0.266028106354418,0.268223994946335,2.19974815286627 +202,5,5,-0.266028106354418,0.298937620984014,3.08651582214762 +202,6,5,-0.266028106354418,1.19175101086346,5.35184933489553 +203,1,5,0.0257161972054362,-0.00777182454554659,0.887316769481998 +203,2,5,0.0257161972054362,0.195171140233948,-2.34645529619854 +203,3,5,0.0257161972054362,0.943185858944576,-0.886172854562971 +203,4,5,0.0257161972054362,0.97297438051478,0.999551256331918 +203,5,5,0.0257161972054362,1.5413637280623,1.46967773439603 +203,6,5,0.0257161972054362,1.09237538286599,4.74829328190536 +204,1,5,0.116551434634878,0.778053422766819,0.923901269733919 +204,2,5,0.116551434634878,0.283905891981172,0.912512995323872 +204,3,5,0.116551434634878,0.939725254375677,1.49793081521857 +204,4,5,0.116551434634878,-0.105034571045226,1.4170606846868 +204,5,5,0.116551434634878,0.0742100832343512,2.88003343428499 +204,6,5,0.116551434634878,2.19181543302645,4.89563372429911 +205,1,5,-0.553125937781561,0.0859988450814071,-0.126962865847536 +205,2,5,-0.553125937781561,-0.113794244285939,1.04790901416167 +205,3,5,-0.553125937781561,1.50265709885282,-0.0121647533712959 +205,4,5,-0.553125937781561,0.753642118076197,-1.87801353698332 +205,5,5,-0.553125937781561,0.798974629882657,1.81444078247793 +205,6,5,-0.553125937781561,1.22940991518573,2.72413883940338 +206,1,5,0.756255260174997,1.37286170808534,-0.0618669650195494 +206,2,5,0.756255260174997,0.0270268243862444,3.87004440747301 +206,3,5,0.756255260174997,0.100304660700929,2.62002685969236 +206,4,5,0.756255260174997,1.50297292169296,2.10325982384873 +206,5,5,0.756255260174997,1.47481298806093,5.08851039988119 +206,6,5,0.756255260174997,1.49517633071853,5.329207300942 +207,1,5,0.402919292696054,0.448184026029819,1.6851353088623 +207,2,5,0.402919292696054,-0.309345001830238,-0.0480430976597863 +207,3,5,0.402919292696054,1.22136648402251,1.95683410550332 +207,4,5,0.402919292696054,1.38816029165409,1.56554848066498 +207,5,5,0.402919292696054,1.12448389959517,4.80323957292905 +207,6,5,0.402919292696054,0.670591444723001,7.61710463034625 +208,1,5,-0.269406763993592,0.206667648141733,2.22207660012001 +208,2,5,-0.269406763993592,0.0288576177863964,1.86463949430405 +208,3,5,-0.269406763993592,1.3635139605955,2.09011945100763 +208,4,5,-0.269406763993592,1.17538271934532,1.67784711763015 +208,5,5,-0.269406763993592,0.615716880287763,4.45742510031184 +208,6,5,-0.269406763993592,0.974785266523716,3.3847857749142 +209,1,5,1.37322747966558,1.51005515038624,-1.95142063469521 +209,2,5,1.37322747966558,0.66548558946035,-0.00933320143897454 +209,3,5,1.37322747966558,0.368067356457127,1.92149373004633 +209,4,5,1.37322747966558,2.2044979582161,2.22552051395123 +209,5,5,1.37322747966558,1.32532189498429,4.42699163117127 +209,6,5,1.37322747966558,2.29794574303181,6.54842437979203 +210,1,5,-1.21546340892904,-1.09510776302794,2.16313335716541 +210,2,5,-1.21546340892904,-0.0869249651887154,-0.113012776498633 +210,3,5,-1.21546340892904,-0.279230661394171,1.56240828061579 +210,4,5,-1.21546340892904,-0.049521274309301,4.51715372142149 +210,5,5,-1.21546340892904,0.00657525039390539,2.86563277524601 +210,6,5,-1.21546340892904,-0.0645347156398759,2.47481350145028 +211,1,5,-0.0701117965547371,-0.715929786867697,0.0255299412037402 +211,2,5,-0.0701117965547371,1.36168105909399,0.998667609709119 +211,3,5,-0.0701117965547371,-0.135807675651325,1.56559791547375 +211,4,5,-0.0701117965547371,1.51851847154815,1.02220638575337 +211,5,5,-0.0701117965547371,0.165993948214305,4.03246492986913 +211,6,5,-0.0701117965547371,0.959425763838153,3.69187032029544 +212,1,5,1.52424799429573,0.754050279225861,1.82652570977404 +212,2,5,1.52424799429573,1.36008329509545,1.87592392729932 +212,3,5,1.52424799429573,1.75505071496496,4.67880928526713 +212,4,5,1.52424799429573,1.22602677201853,6.36476797191923 +212,5,5,1.52424799429573,1.44265784000527,6.45907084431469 +212,6,5,1.52424799429573,1.81542507698388,8.88774350019465 +213,1,5,-1.13898322177537,-0.704850530259102,0.989954312743611 +213,2,5,-1.13898322177537,0.269910398419923,2.64079703302344 +213,3,5,-1.13898322177537,-0.30661493225157,0.234400241959169 +213,4,5,-1.13898322177537,0.173200709565968,0.115256042285581 +213,5,5,-1.13898322177537,-0.174767967493852,2.37243004034063 +213,6,5,-1.13898322177537,0.062705181813378,3.15196239199807 +214,1,5,0.0136635721568709,0.281286858491064,0.308737894502187 +214,2,5,0.0136635721568709,0.709632912619912,2.86447518064835 +214,3,5,0.0136635721568709,0.885759818762133,1.60433385861157 +214,4,5,0.0136635721568709,0.508252732894141,2.9767702147517 +214,5,5,0.0136635721568709,0.995181587793084,4.04309942636804 +214,6,5,0.0136635721568709,0.952559089661326,4.772002205928 +215,1,5,-0.524976137743338,-0.608745128024079,-0.623987422658077 +215,2,5,-0.524976137743338,0.0441390976086862,0.3386677182339 +215,3,5,-0.524976137743338,1.013040638404,0.859440864419736 +215,4,5,-0.524976137743338,0.368850033899774,-0.818130100579811 +215,5,5,-0.524976137743338,1.28996274747004,0.747054019336613 +215,6,5,-0.524976137743338,0.521169572898015,1.66670573365348 +216,1,5,0.528119708420632,0.651477084437435,-0.227441920101093 +216,2,5,0.528119708420632,0.581493185777966,0.0857339576458841 +216,3,5,0.528119708420632,1.74913980629871,0.254230180328475 +216,4,5,0.528119708420632,0.897716621442405,2.42565796191354 +216,5,5,0.528119708420632,1.71655243592702,2.22339067686188 +216,6,5,0.528119708420632,1.34115667416258,4.07122551146554 +217,1,5,-0.278139356660236,-0.432553538863004,-0.781011633085371 +217,2,5,-0.278139356660236,0.422378930946401,-0.567678688198085 +217,3,5,-0.278139356660236,0.562491377327001,-0.965369032587661 +217,4,5,-0.278139356660236,0.500297950693926,0.934629473125308 +217,5,5,-0.278139356660236,1.31421861315264,0.758231973389853 +217,6,5,-0.278139356660236,1.35139823865584,2.03912728069027 +218,1,5,0.49043594073852,0.321110560676538,0.510384585688465 +218,2,5,0.49043594073852,0.706729675579629,-1.04299998018294 +218,3,5,0.49043594073852,0.880495568887448,1.15469874941526 +218,4,5,0.49043594073852,0.865795526845137,2.38447041558231 +218,5,5,0.49043594073852,1.36781795204138,3.44280884506035 +218,6,5,0.49043594073852,2.36944881552513,8.16161080730789 +219,1,5,-1.98958128344518,-0.534903390163236,1.42588349306103 +219,2,5,-1.98958128344518,-0.373290925434039,1.36822515439796 +219,3,5,-1.98958128344518,0.158331617880881,-0.926249260297009 +219,4,5,-1.98958128344518,-0.607765725191721,-1.20272573134459 +219,5,5,-1.98958128344518,0.499900456385467,-0.65847289715537 +219,6,5,-1.98958128344518,-0.128902846011836,2.52464534710552 +220,1,5,-0.213493340618975,0.185391211106394,1.26700247541726 +220,2,5,-0.213493340618975,0.0140852828787207,-1.4427375813286 +220,3,5,-0.213493340618975,0.0174615772931458,-0.306210357001648 +220,4,5,-0.213493340618975,0.539954347150334,1.81353252436983 +220,5,5,-0.213493340618975,0.663050221600149,3.42996813535437 +220,6,5,-0.213493340618975,0.611350423560622,4.1990652063131 +221,1,5,-0.877521252652733,-0.558494533889764,-1.24506994830942 +221,2,5,-0.877521252652733,-0.054670778540151,-1.05245973258249 +221,3,5,-0.877521252652733,-0.434196160948057,-0.49282114431209 +221,4,5,-0.877521252652733,0.178911659606747,-2.59280555529188 +221,5,5,-0.877521252652733,0.68317178405947,1.53212805684875 +221,6,5,-0.877521252652733,1.00294622561415,2.4801642646124 +222,1,5,1.81905705063474,1.37293696497615,1.80371980454759 +222,2,5,1.81905705063474,1.72673008537909,2.32717660429447 +222,3,5,1.81905705063474,1.37469608885288,2.83327951835427 +222,4,5,1.81905705063474,1.44072915243679,5.45277832067784 +222,5,5,1.81905705063474,1.94592738842887,8.26489174542936 +222,6,5,1.81905705063474,2.65622088626748,8.25819620853794 +223,1,5,0.852925428022894,1.67333324392399,0.922675753730593 +223,2,5,0.852925428022894,1.16539499869153,3.04908307330675 +223,3,5,0.852925428022894,1.73698490260616,1.69916384611494 +223,4,5,0.852925428022894,1.13029237034851,4.50861305072888 +223,5,5,0.852925428022894,0.725190204392934,4.0963264949155 +223,6,5,0.852925428022894,1.49173024477212,7.68586493395678 +224,1,5,-0.517878257327904,-1.35732639663923,-1.23119186049687 +224,2,5,-0.517878257327904,1.11491866568095,-0.315709039321787 +224,3,5,-0.517878257327904,0.587780945805903,-0.369763432693986 +224,4,5,-0.517878257327904,0.927163157395854,-0.506417691026338 +224,5,5,-0.517878257327904,0.488711607226898,0.686305714138264 +224,6,5,-0.517878257327904,0.385312701261675,3.62404644990348 +225,1,5,-1.11257146628431,-0.475426417311899,-1.04280843910047 +225,2,5,-1.11257146628431,-1.18634048699766,-2.30358967343468 +225,3,5,-1.11257146628431,0.091173974222244,0.945624995795265 +225,4,5,-1.11257146628431,0.238514872098877,-1.99598194201174 +225,5,5,-1.11257146628431,0.425818445029469,1.50873654886708 +225,6,5,-1.11257146628431,1.05295883081395,2.07497420733641 +226,1,5,-1.81489141645341,-1.24259664095849,1.51719448330531 +226,2,5,-1.81489141645341,-0.554428904590342,1.41446259490764 +226,3,5,-1.81489141645341,-0.299740806185905,2.18468422027469 +226,4,5,-1.81489141645341,0.542734837538821,0.163871949464976 +226,5,5,-1.81489141645341,-0.298329304331962,0.435908366646806 +226,6,5,-1.81489141645341,0.414989726238207,1.96641824618135 +227,1,5,-0.260227815842348,-1.03005150290773,-0.019407951568893 +227,2,5,-0.260227815842348,0.468446628646404,0.0285719838885757 +227,3,5,-0.260227815842348,0.470855524402432,-0.174855338773838 +227,4,5,-0.260227815842348,1.44176617213929,0.69418426946141 +227,5,5,-0.260227815842348,0.226607556826976,4.60969384532652 +227,6,5,-0.260227815842348,0.906000249868074,3.70417043721014 +228,1,5,-0.248887440272954,-0.418765638490002,-0.0863136410068941 +228,2,5,-0.248887440272954,0.314102203471253,2.38681030771552 +228,3,5,-0.248887440272954,0.362571171005034,2.76383374351119 +228,4,5,-0.248887440272954,0.47788363062241,3.55624065831369 +228,5,5,-0.248887440272954,0.520531301129979,3.77951398822541 +228,6,5,-0.248887440272954,1.24345219848178,4.81422068100672 +229,1,5,-0.300211489798693,-0.603458799036538,0.129483577367022 +229,2,5,-0.300211489798693,0.379696846594982,2.39422380550729 +229,3,5,-0.300211489798693,0.285898323584938,0.56453671209308 +229,4,5,-0.300211489798693,-0.343957165096895,1.81928926476601 +229,5,5,-0.300211489798693,1.34665793798712,1.60250849670397 +229,6,5,-0.300211489798693,1.38600178614644,4.44490888146164 +230,1,5,2.18079606684413,0.0328589209121795,1.38593922837778 +230,2,5,2.18079606684413,2.20428194064908,3.38086181845178 +230,3,5,2.18079606684413,1.69611948536251,3.41464459135443 +230,4,5,2.18079606684413,2.64727843019363,4.4219208488337 +230,5,5,2.18079606684413,2.91539831678562,6.44465284207725 +230,6,5,2.18079606684413,2.02577390016517,10.3074908553895 +231,1,5,0.682258613138577,0.232553052940103,0.575192037116431 +231,2,5,0.682258613138577,0.934575833052082,2.93725402467604 +231,3,5,0.682258613138577,1.79765272992026,4.02869272214191 +231,4,5,0.682258613138577,0.700136142980352,5.18621671240476 +231,5,5,0.682258613138577,1.29305205341976,7.67676120198773 +231,6,5,0.682258613138577,1.47254977346884,6.40465083371395 +232,1,5,0.738564646474746,0.255708737415919,2.17066071793656 +232,2,5,0.738564646474746,1.30570948732827,1.61187659853227 +232,3,5,0.738564646474746,1.3389155585084,3.07698051626951 +232,4,5,0.738564646474746,2.51777450060082,3.29898038024719 +232,5,5,0.738564646474746,2.01416388727751,4.04999085321885 +232,6,5,0.738564646474746,1.14429575274794,6.22284218610352 +233,1,5,1.31891652792444,0.943432331705388,-0.0289136497162035 +233,2,5,1.31891652792444,0.978239063144288,2.06170299473121 +233,3,5,1.31891652792444,0.310622694521734,3.69752660270523 +233,4,5,1.31891652792444,0.68052796604515,1.5204378915292 +233,5,5,1.31891652792444,1.70099796725061,5.71280736379098 +233,6,5,1.31891652792444,2.00471022400003,5.58278554448763 +234,1,5,1.28907990310246,1.65412984713345,-0.602892382604196 +234,2,5,1.28907990310246,0.853258601018456,0.907141408866408 +234,3,5,1.28907990310246,1.60150771388762,3.24796159869254 +234,4,5,1.28907990310246,1.96933258012763,4.19436697567741 +234,5,5,1.28907990310246,1.27632561755083,4.99253953190101 +234,6,5,1.28907990310246,1.77582680830315,4.8548769273247 +235,1,5,0.978070722904636,0.948150182775719,-1.03874117061971 +235,2,5,0.978070722904636,0.543444277286829,0.925766870985002 +235,3,5,0.978070722904636,1.49477068531528,2.21839953506068 +235,4,5,0.978070722904636,1.56819489132473,1.93251425438558 +235,5,5,0.978070722904636,1.59527502367316,3.20318666486669 +235,6,5,0.978070722904636,1.35543245600248,6.86424987830461 +236,1,5,0.00643884641450284,-0.641212010033781,-0.575014663395219 +236,2,5,0.00643884641450284,0.937855659547919,2.06384914461294 +236,3,5,0.00643884641450284,0.739732482846854,1.41647034926797 +236,4,5,0.00643884641450284,1.10190774770209,3.79348717413345 +236,5,5,0.00643884641450284,1.02193117870911,3.04259873537341 +236,6,5,0.00643884641450284,0.0204587745911464,7.00891684792682 +237,1,5,0.0165814801249611,-0.305408606389655,1.76392967553631 +237,2,5,0.0165814801249611,0.267051028226902,-1.10004204212255 +237,3,5,0.0165814801249611,0.319994248367331,2.40546891854285 +237,4,5,0.0165814801249611,1.43776249361238,2.12131653600467 +237,5,5,0.0165814801249611,1.14569000055294,2.5595375324857 +237,6,5,0.0165814801249611,0.435442855090185,4.89078063646968 +238,1,5,1.91587682170855,1.28061275956914,-0.647304703918855 +238,2,5,1.91587682170855,0.682339854322889,1.91715709450631 +238,3,5,1.91587682170855,0.682233140309582,2.42845110380224 +238,4,5,1.91587682170855,1.19451909853685,5.00427468249585 +238,5,5,1.91587682170855,1.36885315652453,5.70056465024374 +238,6,5,1.91587682170855,1.52792382399838,8.49809489229398 +239,1,5,-1.76542576063516,-1.30871766053306,0.221759257176214 +239,2,5,-1.76542576063516,-0.706967060937396,-0.283031516205847 +239,3,5,-1.76542576063516,-0.0826534122809553,-0.199081921554736 +239,4,5,-1.76542576063516,0.553569425457922,0.804541456652704 +239,5,5,-1.76542576063516,-0.425046079826805,0.893639807836897 +239,6,5,-1.76542576063516,1.118581133231,2.48466150451466 +240,1,5,-1.47447432420743,-0.590776716929888,-2.04062301930976 +240,2,5,-1.47447432420743,-0.0386169438484504,-1.89717081094193 +240,3,5,-1.47447432420743,0.210964261447659,1.17208473689677 +240,4,5,-1.47447432420743,-0.640970403235413,-1.29107091503318 +240,5,5,-1.47447432420743,0.758534883730153,0.323916789750266 +240,6,5,-1.47447432420743,1.2629442004812,1.9383167523482 +241,1,5,0.444480192099237,0.618633887994417,-2.68240112037008 +241,2,5,0.444480192099237,0.386435895591268,-2.42533638325667 +241,3,5,0.444480192099237,0.0306556682985822,-0.208110794073889 +241,4,5,0.444480192099237,1.65754806459742,1.05287475605699 +241,5,5,0.444480192099237,1.65965765467233,3.85979072939338 +241,6,5,0.444480192099237,0.390040101044708,5.13972097810125 +242,1,5,-0.109370786918,0.180835200749669,0.469693708300177 +242,2,5,-0.109370786918,-0.241649823954673,1.68019556392288 +242,3,5,-0.109370786918,1.63054915289421,2.99946048149086 +242,4,5,-0.109370786918,1.79876724716694,1.70281395029541 +242,5,5,-0.109370786918,1.50933658861389,1.92243530107267 +242,6,5,-0.109370786918,1.44996003366407,3.62636572590989 +243,1,5,0.336040969026486,0.395984087004319,-0.510470501592253 +243,2,5,0.336040969026486,1.49213107911934,0.341057260343169 +243,3,5,0.336040969026486,1.0322243877222,1.5963528291938 +243,4,5,0.336040969026486,1.13980145115043,3.22875490267963 +243,5,5,0.336040969026486,1.02657502168611,2.99297209111054 +243,6,5,0.336040969026486,1.51889009798494,5.66833238619657 +244,1,5,0.0246226254591995,-0.310292434088861,0.138069972547055 +244,2,5,0.0246226254591995,0.141199815446702,1.29756197057001 +244,3,5,0.0246226254591995,0.643356826099255,1.84884603919037 +244,4,5,0.0246226254591995,1.15478197762161,1.39675030805328 +244,5,5,0.0246226254591995,0.339075524589951,4.5868265068948 +244,6,5,0.0246226254591995,1.31437861623794,5.24668297480788 +245,1,5,0.885781948931482,0.806314046758624,2.2654923874529 +245,2,5,0.885781948931482,1.18408803633258,3.43352381181165 +245,3,5,0.885781948931482,0.227934592981508,3.16745054969171 +245,4,5,0.885781948931482,1.55756174226992,3.35340671191667 +245,5,5,0.885781948931482,2.33813525718331,4.78860640648161 +245,6,5,0.885781948931482,1.99813377457044,6.51661354997833 +246,1,5,-0.722659484881553,-0.384792945271069,-1.98735622308168 +246,2,5,-0.722659484881553,0.314472242298869,-0.285445896415748 +246,3,5,-0.722659484881553,1.11495488518433,-1.08780307763077 +246,4,5,-0.722659484881553,0.467747700320704,0.240682059810906 +246,5,5,-0.722659484881553,1.33148113439858,0.926984623451972 +246,6,5,-0.722659484881553,1.01374841804555,2.50113616748474 +247,1,5,1.59162124543521,0.801382985950288,0.157749357181468 +247,2,5,1.59162124543521,1.08447653818498,0.966600766348528 +247,3,5,1.59162124543521,1.64504914121795,2.97737955065785 +247,4,5,1.59162124543521,1.66859567663211,4.27374442535286 +247,5,5,1.59162124543521,1.20129706760012,3.93965864036669 +247,6,5,1.59162124543521,1.85582140799441,7.47447291305973 +248,1,5,0.00628203166510098,0.967574078834003,-0.277598332529191 +248,2,5,0.00628203166510098,0.251219158257299,-1.35116285419275 +248,3,5,0.00628203166510098,0.37795289944178,0.394806803517492 +248,4,5,0.00628203166510098,1.38110864533361,0.390223429962069 +248,5,5,0.00628203166510098,2.14074652648849,3.0551896743306 +248,6,5,0.00628203166510098,1.2842173284638,5.41393151214885 +249,1,5,-0.157830993412902,0.225604784343245,-0.63926524186835 +249,2,5,-0.157830993412902,0.815653032758968,1.08592736946653 +249,3,5,-0.157830993412902,-0.926936488629974,0.70719821540352 +249,4,5,-0.157830993412902,0.99159892459773,3.42146939341312 +249,5,5,-0.157830993412902,1.55574470790068,5.93041240162439 +249,6,5,-0.157830993412902,1.14913138457702,3.7143475809707 +250,1,5,-0.486593733281175,0.496828444822754,0.484715409441276 +250,2,5,-0.486593733281175,0.050098683873394,3.15320629989067 +250,3,5,-0.486593733281175,0.250532784667647,1.5334139004638 +250,4,5,-0.486593733281175,-1.07335766406325,-0.164378577396767 +250,5,5,-0.486593733281175,0.0773300025131354,3.30694573947018 +250,6,5,-0.486593733281175,1.12906863829978,4.46490543466803 diff --git a/changelog.d/20260831-twfe-weight-diagnostics.md b/changelog.d/20260831-twfe-weight-diagnostics.md new file mode 100644 index 000000000..ab016915f --- /dev/null +++ b/changelog.d/20260831-twfe-weight-diagnostics.md @@ -0,0 +1,24 @@ +### Added +- **TWFE weight diagnostics** (port of Brantly Callaway's `twfeweights` R + package, MIT): what a two-way fixed effects regression *implicitly* weights + on staggered-adoption data. + - `attgt_weights(results, aggregation="twfe"|"overall"|"simple")` reports the + weight a TWFE regression, ATT^O, or ATT^simple places on each ATT(g,t), + plus post-period negative-weight counts. Returns `ATTGTWeightsResult`. + The CS estimands honour the fit's `anticipation` window (an explicit + `anticipation=` on the frame path); `"twfe"` keeps `t >= g`. + - `decompose_twfe_weights(data, ..., method="fwl")` re-derives the estimate + from its ATT(g,t) building blocks and returns `TWFEDecompositionResult` + with `pre_period_contribution` - the sample contribution of the + pre-treatment cells, which can reflect parallel-trends violations or + sampling variation - and, with `balance_covariates=`, implicit-weight + covariate balance. `plot_twfe_weights()` renders either view (matplotlib + or plotly). + - Validation: rejects NaN / `-inf` cohort labels, covariate-adjusted fits + under `aggregation="twfe"`, unbalanced panels, non-finite outcomes / + covariates, duplicated or non-finite ATT(g,t) cells, an incomplete + group-time grid, and invalid sampling weights. Two structural gaps are + handled as R does instead of raising: a cohort with no estimable post cell + is dropped (`did`'s first-period drop, or running out of comparison + units), and under `control_group="not_yet_treated"` the CS estimands + average over each cohort's available post periods (`aggte`). diff --git a/diff_diff/__init__.py b/diff_diff/__init__.py index 1ef3b75f6..dcbb3ea77 100644 --- a/diff_diff/__init__.py +++ b/diff_diff/__init__.py @@ -298,6 +298,14 @@ TROPResults, trop, ) +from diff_diff.twfe_weights import ( + attgt_weights, + decompose_twfe_weights, +) +from diff_diff.twfe_weights_results import ( + ATTGTWeightsResult, + TWFEDecompositionResult, +) from diff_diff.two_stage import ( TwoStageBootstrapResults, TwoStageDiD, @@ -323,6 +331,7 @@ plot_sensitivity, plot_staircase, plot_synth_weights, + plot_twfe_weights, ) from diff_diff.wooldridge import WooldridgeDiD from diff_diff.wooldridge_results import WooldridgeDiDResults @@ -459,6 +468,13 @@ def __getattr__(name: str) -> _Any: "TWFEWeightsResult", "chaisemartin_dhaultfoeuille", "twowayfeweights", + # TWFE weight diagnostics (Callaway `twfeweights` port) - distinct from + # the dCDH `twowayfeweights` surface above: these weight ATT(g,t) + # parameters, not (unit, time) cells. + "ATTGTWeightsResult", + "TWFEDecompositionResult", + "attgt_weights", + "decompose_twfe_weights", # WooldridgeDiD (ETWFE) "WooldridgeDiD", "WooldridgeDiDResults", @@ -479,6 +495,7 @@ def __getattr__(name: str) -> _Any: "DurationDiDPretestResults", # Visualization "plot_bacon", + "plot_twfe_weights", "plot_event_study", "plot_group_effects", "plot_sensitivity", diff --git a/diff_diff/dml_did.py b/diff_diff/dml_did.py index 16e6f4bdf..bb2d17a15 100644 --- a/diff_diff/dml_did.py +++ b/diff_diff/dml_did.py @@ -3362,6 +3362,7 @@ def fit( # stays admitted). is_survey_fit=survey_metadata is not None, bootstrap_results=bootstrap_results, + covariates=covariates, ) self.results_ = results self.is_fitted_ = True diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index b84b3f03d..36c5289b5 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -1534,6 +1534,60 @@ results.print_summary() plot_bacon(results) ``` +### TWFE Weight Diagnostics + +What a TWFE regression implicitly weights on staggered data. Distinct from +`twowayfeweights` (dCDH), which weights (unit, time) cells: these weight +ATT(g,t) parameters. Ported from Brantly Callaway's `twfeweights` R package +(MIT); methodology Baker, Callaway, Cunningham, Goodman-Bacon & Sant'Anna +(2025). + +```python +attgt_weights( + results, # CallawaySantAnnaResults, or a (g,t) frame + aggregation="twfe", # "twfe" | "overall" (ATT^O) | "simple" + data=None, unit=None, time=None, first_treat=None, # frame path only + weights=None, # unit-level sampling weights +) -> ATTGTWeightsResult + +decompose_twfe_weights( + data, # balanced long panel (it re-estimates) + outcome=, unit=, time=, first_treat=, + method="fwl", + covariates=None, + base_period="first_period", # or "gmin1" + balance_covariates=None, # enables result.covariate_balance() + weights=None, +) -> TWFEDecompositionResult + +plot_twfe_weights(result, kind="auto") # "weights" | "balance" +``` + +`aggregation="twfe"` requires a fit with `base_period="universal"`, +`control_group="never_treated"` AND no covariates (R twfe_weights' three +restrictions); it raises otherwise. ATT^O and ATT^simple weights are +non-negative and sum to one, so comparing `implied_att` across the three +aggregations shows what the TWFE specification costs. + +Both entry points fail closed on input R never faced: NaN / `-inf` cohort +labels (never-treated is exactly `0` or `+inf`), duplicated or non-finite +ATT(g,t) cells, an incomplete group-time grid (`"twfe"` needs every cohort x +period cell, the CS estimands every post cell), and sampling weights that are +not finite, non-negative and positive-mass. Two structural gaps mirror R +rather than raising, each with a `UserWarning`: a cohort with no estimable +post cell is dropped from the table and the cohort shares (`did`'s +first-period drop), and under `control_group="not_yet_treated"` the cells CS +marks `zero_treated_control` are treated as structurally absent, so +`"overall"`/`"simple"` average over each cohort's AVAILABLE post periods +(`aggte`). `n_negative_post` / `negative_post_weight_share` report the +pathology (negative weight on POST cells); `n_negative` counts pre cells too, +and is near-half in every staggered design because the TWFE weights sum to +zero over the full grid. + +`decompose_twfe_weights` takes the raw panel rather than a fitted result +because it re-estimates. It is tied to `attgt_weights` by an identity: +`attgt_weights(cs, aggregation="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate`. + ### StaggeredTripleDifference DEPRECATED in 3.9, removed in 4.0 (ledger row M-013). Use @@ -2017,6 +2071,34 @@ Returned by `BaconDecomposition.fit()` (and the deprecated `bacon_decompose()` w **Methods:** `summary()`, `print_summary()`, `to_dataframe()` +### ATTGTWeightsResult + +Diagnostic result from `attgt_weights`. No inference quintet - the +decomposition is an algebraic identity. + +- `weights`: DataFrame with `group`, `time`, `post`, `weight`, `att` +- `implied_att`: `sum(weight * att)` - the TWFE coefficient when + `aggregation="twfe"` +- `n_negative`, `negative_weight_share`: the staggered-TWFE pathology +- `aggregation`, `source`, `control_group`, `base_period`, `n_cells` +- `summary()`, `to_dataframe()`, `to_dict()` + +### TWFEDecompositionResult + +Diagnostic result from `decompose_twfe_weights`. + +- `cells`: DataFrame with `group`, `time`, `post`, `att`, `weight`, `ess`, + `remainder` +- `estimate` == `decomposition` + `remainder` +- `pre_period_contribution`: sample contribution of the PRE-treatment cells + (can reflect differential pre-trends OR sampling variation; diagnostic + evidence, not proof the post-treatment assumption fails) +- `post_only`, `effective_sample_size`, `covariates`, `base_period` +- `covariate_balance(level="summary"|"cell", standardize=True, + post_only=True)`: implicit-weight covariate balance; raises when + `balance_covariates=` was not requested +- `summary()`, `to_dataframe()`, `to_dict()` + ### Comparison2x2 Individual 2x2 DiD comparison (used in BaconDecompositionResults). @@ -2532,6 +2614,28 @@ plot_bacon( ) ``` +### plot_twfe_weights + +```python +from diff_diff import plot_twfe_weights + +plot_twfe_weights( + results, # ATTGTWeightsResult | TWFEDecompositionResult + kind="auto", # "weights" | "balance" ("auto" picks balance + # when the result carries a balance table) + standardize=True, absolute_value=True, # balance view + annotate=False, ax=None, show=True, + backend="matplotlib", # or "plotly" +) +``` + +`kind="weights"` scatters weight against ATT(g,t), coloured by pre/post - points +left of the vertical zero line carry negative weight; the title counts POST-only +negatives. `kind="balance"` scatters unweighted against implicitly-weighted +covariate differences; points near the horizontal axis are covariates the +implicit weights balance. With `absolute_value=False` the reference diagonal +spans `[-limit, limit]` so signed differences read correctly. + ### plot_power_curve ```python diff --git a/diff_diff/guides/llms.txt b/diff_diff/guides/llms.txt index a9df7dc97..096f2b6a3 100644 --- a/diff_diff/guides/llms.txt +++ b/diff_diff/guides/llms.txt @@ -92,6 +92,7 @@ The site is organized into 5 sections, each with a landing page: - [Manipulation Testing](https://diff-diff.readthedocs.io/en/stable/api/regression_discontinuity.html): Cattaneo, Jansson & Ma (2020) density-discontinuity manipulation test (`RDDensityTest`), parity with R rddensity 3.0 - boundary-adaptive local polynomial density estimation at the cutoff, robust bias-corrected inference, unrestricted/restricted models, jackknife/plugin variances, data-driven bandwidths, mass-point adjustment - [Parallel Trends Testing](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html): Simple and Wasserstein-robust parallel trends tests, equivalence testing (TOST) - [Placebo Tests](https://diff-diff.readthedocs.io/en/stable/api/diagnostics.html): Placebo timing, group, permutation, and leave-one-out diagnostics +- [TWFE Weight Diagnostics](https://diff-diff.readthedocs.io/en/stable/api/twfe_weights.html): Baker et al. (2025) implicit weights on ATT(g,t) - `attgt_weights(results, aggregation='twfe'|'overall'|'simple')` takes a fitted `CallawaySantAnnaResults` (raw ATT(g,t) frame + panel as fallback) and returns the weight each estimand places on each group-time effect, with the negative-weight share; `decompose_twfe_weights(data, outcome=, unit=, time=, first_treat=, method='fwl', covariates=)` re-derives the TWFE estimate from its ATT(g,t) building blocks with `pre_period_contribution` (the pre-treatment cells' sample contribution, which can be pre-trends or sampling noise), and `result.covariate_balance()` reports implicit-weight covariate balance. Plot with `plot_twfe_weights`. R `twfeweights` 0.9.0 output parity - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html): Rambachan & Roth (2023) sensitivity analysis — robust CI under parallel trends violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html): Roth (2022) Section II.A-B no-individually-significant (NIS) box-probability pretest power + minimum detectable violation; `pretest_form='nis'` (default) implements the paper's primary form, `pretest_form='wald'` retained as paper-supported alternative (Propositions 1+3+4 all apply); linear-violation MDV in Roth's γ units when relative-time labels are threaded through `fit()`; full Σ_22 routing on non-bootstrap CallawaySantAnna and SunAbraham adapters and on admitted CS-/StackedDiD-sourced `aggregate('event_study')` containers (StackedDiD persists its ES VCV in every inference mode) - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html): Analytical and simulation-based power analysis — MDE, sample size, power curves for study design diff --git a/diff_diff/staggered.py b/diff_diff/staggered.py index 37a544907..134acdafb 100644 --- a/diff_diff/staggered.py +++ b/diff_diff/staggered.py @@ -7,7 +7,7 @@ import bisect import warnings -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple import numpy as np import pandas as pd @@ -3056,6 +3056,7 @@ def fit( group_time_effects, is_survey_fit=survey_metadata is not None, bootstrap_results=bootstrap_results, + covariates=covariates, ) self.is_fitted_ = True @@ -5151,6 +5152,7 @@ def _build_aggregation_kit( *, is_survey_fit: bool = False, bootstrap_results: Optional["CSBootstrapResults"] = None, + covariates: Optional[Sequence[str]] = None, ) -> Optional["AggregationKit"]: """Distil the fit-time state post-fit re-aggregation needs. @@ -5185,6 +5187,16 @@ def _build_aggregation_kit( # (or DDD) survey fit does not warn as "CallawaySantAnna" on post-fit # aggregate(). Legacy kits without the key default at the read site. bookkeeping["bootstrap_label"] = getattr(estimator, "_BOOTSTRAP_LABEL", "CallawaySantAnna") + # Covariate usage, recorded so downstream diagnostics can refuse designs + # their formulas do not cover (``attgt_weights(aggregation="twfe")`` + # mirrors R twfe_weights' ``xformla == ~1`` restriction). Column NAMES + # only - never values - so the data-minimization contract holds. + bookkeeping["covariates"] = tuple(covariates or ()) + # Panel balance, recorded so ``attgt_weights`` can reject an unbalanced + # fitted result: its cohort shares and E_t[D] assume the same units in + # every period. Defaults True when the key is absent (a legacy kit, or a + # producer that never computed it). + bookkeeping["is_balanced"] = bool((precomputed or {}).get("is_balanced", True)) # Data minimization: the results object is picklable and users share # result artifacts, so the kit must not turn it into a carrier for raw diff --git a/diff_diff/twfe_weights.py b/diff_diff/twfe_weights.py new file mode 100644 index 000000000..7c1cda31f --- /dev/null +++ b/diff_diff/twfe_weights.py @@ -0,0 +1,1788 @@ +"""Implicit TWFE weights on group-time average treatment effects. + +A two-way fixed effects regression run on staggered-adoption data does not +estimate a simple average of the underlying ATT(g, t). It estimates a +*weighted* average, and some of those weights can be negative - so the +coefficient need not lie in the convex hull of the effects it summarizes. +:func:`attgt_weights` reports those weights, next to the weights the target +estimands ATT^O and ATT^simple would use. :func:`decompose_twfe_weights` +re-derives the regression from its building blocks and separates the part +driven by pre-treatment parallel-trends violations. + +Distinct from :func:`diff_diff.twowayfeweights`, which implements the de +Chaisemartin & D'Haultfoeuille (2020) Theorem 1 decomposition: that one +weights ``(unit, time)`` cells, this one weights ATT(g, t) *parameters*. +Distinct also from :class:`diff_diff.BaconDecomposition`, which decomposes +TWFE into 2x2 DiD comparisons rather than into group-time effects. + +Ported from the R package ``twfeweights`` (version 0.9.0) by Brantly +Callaway, released under the MIT License. The upstream notice is reproduced +in full, as its terms require:: + + MIT License + + Copyright (c) 2023 Brantly Callaway + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Methodology: Baker, Callaway, Cunningham, Goodman-Bacon & Sant'Anna (2025), +"Difference-in-Differences Designs: A Practitioner's Guide" +(arXiv:2503.13323); Callaway & Sant'Anna (2021) for the ATT^O / ATT^simple +weights. +""" + +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Union + +import numpy as np +import pandas as pd + +from diff_diff.linalg import solve_ols +from diff_diff.twfe_weights_results import ( + ATTGTWeightsResult, + TWFEDecompositionResult, +) +from diff_diff.utils import within_transform + +if TYPE_CHECKING: # pragma: no cover - typing only + from diff_diff.staggered_results import CallawaySantAnnaResults + +__all__ = ["attgt_weights", "decompose_twfe_weights"] + +_AGGREGATIONS = ("twfe", "overall", "simple") + + +def _is_never(values: np.ndarray) -> np.ndarray: + """Boolean mask for never-treated cohort labels. + + diff-diff and R ``did`` have both used ``0`` and ``+inf`` as the + never-treated sentinel over time; accept exactly those two and normalize + to ``0``. Every OTHER non-finite label (NaN, ``-inf``) is an input error - + see :func:`_validate_cohort_labels` - not a never-treated unit. + """ + arr = np.asarray(values, dtype=float) + return (arr == 0) | (arr == np.inf) + + +def _validate_cohort_labels( + values: np.ndarray, *, unit_ids: Optional[np.ndarray] = None, what: str = "first_treat" +) -> None: + """Reject NaN / ``-inf`` cohort labels instead of silently treating them as never-treated.""" + arr = np.asarray(values, dtype=float) + bad = np.isnan(arr) | (arr == -np.inf) + if bad.any(): + idx = np.flatnonzero(bad)[:5] + who = [unit_ids[i] for i in idx] if unit_ids is not None else idx.tolist() + raise ValueError( + f"{what!r} contains NaN or -inf cohort label(s) for unit(s) {who!r}; " + "never-treated units must be coded exactly 0 or +inf, and every " + "other unit needs a finite first-treatment period" + ) + + +def _validate_time_labels(values: np.ndarray, *, what: str = "time") -> np.ndarray: + """Validate period labels and return them as the canonical numeric key. + + Every downstream step - sorting, reshaping, the positional grid, cohort + mapping - MUST use this one key. Using the raw column instead lets a + numeric-string label ("10" sorts before "2" lexicographically) desynchronize + the sort order from the grid, silently rebuilding a different panel. + """ + arr = pd.to_numeric(pd.Series(np.asarray(values)), errors="coerce").to_numpy(dtype=float) + bad = ~np.isfinite(arr) + if bad.any(): + raise ValueError( + f"{what!r} contains {int(bad.sum())} non-finite or non-numeric period " + f"label(s) (first at row {int(np.flatnonzero(bad)[0])}); every observation " + "must carry a finite period" + ) + return arr + + +def _positional_grid( + time_periods: Sequence[Any], +) -> Dict[float, int]: + """Map ordered period labels onto ``1..T``. + + R computes ``(maxT - g + 1) / length(tlist)`` directly on the raw period + labels, which is only correct when those labels are consecutive integers. + Working in positional time makes the same expression correct on gapped or + non-integer grids, and is bit-identical when the grid IS consecutive + (mpdta's 2003..2007 maps to 1..5 and both give 4/5 for g = 2004). + Recorded as a deviation in the methodology registry. + """ + ordered = sorted({float(t) for t in time_periods}) + return {t: i + 1 for i, t in enumerate(ordered)} + + +def _to_positional_cohort(cohorts: np.ndarray, grid: Dict[float, int]) -> np.ndarray: + """Cohort labels -> positional time; never-treated stays 0. + + Mirrors ``BMisc::orig2t``, which leaves the never-treated sentinel alone + under positional rescaling. + """ + out = np.zeros(len(cohorts), dtype=float) + never = _is_never(cohorts) + for i, (g, is_never) in enumerate(zip(cohorts, never)): + if is_never: + continue + key = float(g) + if key not in grid: + raise ValueError( + f"cohort label {g!r} is not one of the observed time periods " + f"{sorted(grid)!r}; cannot place it on the period grid" + ) + out[i] = grid[key] + return out + + +def _validate_unit_weights( + w: np.ndarray, is_never: np.ndarray, *, require_control_mass: bool +) -> None: + """Shared contract for unit-level sampling weights. + + Finite, non-negative, positive total, positive TREATED mass; positive + never-treated mass only where the never-treated group enters the formula + (``aggregation="twfe"`` and the decomposition) - ATT^O / ATT^simple are + defined without one. + """ + if not np.all(np.isfinite(w)): + raise ValueError("unit weights must be finite; got NaN or infinite weight(s)") + if (w < 0).any(): + idx = np.flatnonzero(w < 0)[:5].tolist() + raise ValueError( + f"unit weights must be non-negative; negative weight(s) at unit index {idx!r}" + ) + if w.sum() <= 0: + raise ValueError("unit weights sum to zero; cannot form cohort shares") + if w[~is_never].sum() <= 0: + raise ValueError( + "the ever-treated units carry zero total weight; cannot form cohort shares" + ) + if require_control_mass and w[is_never].sum() <= 0: + raise ValueError( + "the never-treated comparison group carries zero total weight, so " + "every group-time contrast is undefined" + ) + + +def _cohort_masses( + unit_cohorts: np.ndarray, + grid: Dict[float, int], + weights: Optional[np.ndarray], + *, + require_control_mass: bool = False, +) -> Tuple[Dict[int, float], Dict[int, float], Dict[int, float], float]: + """Cohort shares and treated-share-by-period, all in positional time. + + Returns + ------- + p_all : {positional g: share of ALL units in cohort g} + R's ``pg2`` - the denominator is every unit, never-treated included. + Used by the TWFE weights. + p_treated : {positional g: share of EVER-TREATED units in cohort g} + R's ``pg``. Used by the ATT^O / ATT^simple weights. + e_dt : {positional t: weighted share of units treated by t} + R's ``Edt(t)``. + mean_e_dt : float + R's ``mEdt`` - the average of ``e_dt`` over the period grid. + """ + g_pos = _to_positional_cohort(unit_cohorts, grid) + w = np.ones(len(g_pos)) if weights is None else np.asarray(weights, dtype=float) + if len(w) != len(g_pos): + raise ValueError(f"weights has length {len(w)} but there are {len(g_pos)} units") + _validate_unit_weights(w, g_pos == 0, require_control_mass=require_control_mass) + total = w.sum() + + treated = g_pos != 0 + treated_mass = w[treated].sum() + + cohorts = sorted({int(g) for g in g_pos if g != 0}) + p_all = {g: float(w[g_pos == g].sum() / total) for g in cohorts} + p_treated = {g: float(w[g_pos == g].sum() / treated_mass) for g in cohorts} + + periods = sorted(grid.values()) + e_dt = {t: float(w[treated & (g_pos <= t)].sum() / total) for t in periods} + mean_e_dt = float(np.mean([e_dt[t] for t in periods])) + return p_all, p_treated, e_dt, mean_e_dt + + +def _twfe_weight_vector( + groups: np.ndarray, + times: np.ndarray, + n_periods: int, + p_all: Dict[int, float], + e_dt: Dict[int, float], + mean_e_dt: float, +) -> np.ndarray: + """Weights a static TWFE regression places on each ATT(g, t). + + ``h(g,t) = 1[t >= g] - (maxT - g + 1)/T - E_t[D] + mean_t E_t[D]`` + ``num(g,t) = h(g,t) * p_g``, normalized by the sum over post cells. + + All arguments are in positional time, so ``maxT == n_periods``. + """ + h = ( + (times >= groups).astype(float) + - (n_periods - groups + 1.0) / n_periods + - np.array([e_dt[int(t)] for t in times]) + + mean_e_dt + ) + num = h * np.array([p_all[int(g)] for g in groups]) + post = times >= groups + denom = num[post].sum() + if denom == 0: + raise ValueError( + "TWFE weight normalization is degenerate (post-treatment weights " + "sum to zero); the regression has no identifying variation" + ) + return num / denom + + +def _overall_weight_vector( + groups: np.ndarray, + p_treated: Dict[int, float], + post_mask: np.ndarray, + n_post_available: Dict[int, int], +) -> np.ndarray: + """ATT^O weights: ``1[t >= g - anticipation] * pbar_g / n_post_g``. + + Not renormalized - the per-cohort divisor is each cohort's number of + AVAILABLE post periods, so the weights already sum to one (the + ``pbar_g`` sum to one over cohorts). Counting the available cells rather + than writing ``(maxT - g + 1)`` analytically is what makes the + anticipation window and the structurally-absent ``zero_treated_control`` + cells come out right, and the two agree exactly on a complete grid. + """ + divisor = np.array([float(n_post_available[int(g)]) for g in groups]) + return post_mask.astype(float) * np.array([p_treated[int(g)] for g in groups]) / divisor + + +def _simple_weight_vector( + groups: np.ndarray, + p_treated: Dict[int, float], + post_mask: np.ndarray, +) -> np.ndarray: + """ATT^simple weights: ``1[t >= g - anticipation] * pbar_g``, normalized.""" + raw = post_mask.astype(float) * np.array([p_treated[int(g)] for g in groups]) + total = raw.sum() + if total == 0: + raise ValueError( + "ATT^simple weight normalization is degenerate (no post-treatment " + "cells carry weight)" + ) + return raw / total + + +def _attgt_from_cs( + results: "CallawaySantAnnaResults", +) -> Tuple[pd.DataFrame, Dict[Tuple[Any, Any], Optional[str]]]: + """Extract the ``(g, t, att)`` table from a fitted CS result. + + Non-estimable cells (``skip_reason`` set, NaN effect) are left out of the + table and reported in the returned ``{(g, t): skip_reason}`` map, so the + caller can decide - per aggregation - whether the gap is structural, a + harmless pre-period drop, or a hard error. + """ + rows: List[Dict[str, Any]] = [] + skipped: Dict[Tuple[Any, Any], Optional[str]] = {} + for (g, t), cell in results.group_time_effects.items(): + effect = cell.get("effect", np.nan) + if cell.get("skip_reason") is not None or not np.isfinite(effect): + skipped[(g, t)] = cell.get("skip_reason") + continue + rows.append({"group": g, "time": t, "att": float(effect)}) + if not rows: + raise ValueError( + "the fitted result has no estimable group-time cells; there is nothing to weight" + ) + table = pd.DataFrame(rows).sort_values(["group", "time"]).reset_index(drop=True) + return table, skipped + + +def _attgt_from_frame( + frame: pd.DataFrame, +) -> Tuple[pd.DataFrame, Dict[Tuple[Any, Any], Optional[str]]]: + """Extract ``(g, t, att)`` from a user-supplied ATT(g, t) frame. + + ``effect`` is preferred over ``att`` because that is the column + ``CallawaySantAnnaResults.to_dataframe("group_time")`` emits - so the + fallback consumes our own frame verbatim, including its ``skip_reason`` + column when present. Duplicate cells and non-finite ``group`` / ``time`` + labels are rejected; a non-finite effect is reported in the skip map, not + silently kept (an ``inf`` ATT would otherwise propagate into + ``implied_att``). + """ + missing = {"group", "time"} - set(frame.columns) + if missing: + raise ValueError( + f"ATT(g,t) frame is missing required column(s) {sorted(missing)!r}; " + "expected 'group', 'time', and one of 'effect' / 'att'" + ) + for candidate in ("effect", "att"): + if candidate in frame.columns: + value_col = candidate + break + else: + raise ValueError( + "ATT(g,t) frame must carry an 'effect' or 'att' column; got " f"{list(frame.columns)!r}" + ) + groups = pd.to_numeric(frame["group"], errors="coerce").to_numpy(dtype=float) + times = pd.to_numeric(frame["time"], errors="coerce").to_numpy(dtype=float) + _validate_cohort_labels(groups, what="group") + _validate_time_labels(frame["time"].to_numpy(), what="time") + key = pd.MultiIndex.from_arrays([frame["group"].to_numpy(), frame["time"].to_numpy()]) + if key.duplicated().any(): + dupes = sorted({tuple(k) for k in key[key.duplicated()].tolist()})[:5] + raise ValueError( + f"ATT(g,t) frame has duplicated (group, time) cell(s) {dupes!r}; each " + "cell must appear exactly once" + ) + att = pd.to_numeric(frame[value_col], errors="coerce").to_numpy(dtype=float) + reasons = ( + frame["skip_reason"].tolist() if "skip_reason" in frame.columns else [None] * len(frame) + ) + table = pd.DataFrame( + {"group": frame["group"].to_numpy(), "time": frame["time"].to_numpy(), "att": att} + ) + finite = np.isfinite(att) + skipped: Dict[Tuple[Any, Any], Optional[str]] = {} + for i in np.flatnonzero(~finite): + reason = reasons[i] + skipped[(table["group"].iat[i], table["time"].iat[i])] = ( + None + if reason is None or (isinstance(reason, float) and np.isnan(reason)) + else str(reason) + ) + table = table[finite] + if table.empty: + raise ValueError("ATT(g,t) frame has no finite effects to weight") + _ = groups, times # validated above; positional mapping happens in the caller + return table.sort_values(["group", "time"]).reset_index(drop=True), skipped + + +def _unit_cohorts_from_frame( + data: pd.DataFrame, unit: str, time: str, first_treat: str +) -> Tuple[np.ndarray, np.ndarray, Optional[np.ndarray]]: + """Collapse a long panel to one cohort label per unit.""" + for col in (unit, time, first_treat): + if col not in data.columns: + raise ValueError(f"column {col!r} not found in data") + key = _validate_time_labels(data[time].to_numpy(), what=time) + # Exactly one observation per unit-period, on a rectangular grid: the cohort + # shares and E_t[D] assume the same units in every period, exactly as the + # fitted path does. + cells = pd.MultiIndex.from_arrays([data[unit].to_numpy(), key]) + if cells.duplicated().any(): + raise ValueError( + "data has duplicate (unit, period) row(s); the frame path needs " + "exactly one observation per unit-period" + ) + n_periods = len(pd.unique(key)) + observed = pd.Series(key).groupby(data[unit].to_numpy(), sort=True).nunique() + if not (observed == n_periods).all(): + raise ValueError( + "data is not a balanced panel: some units are missing periods, so " + "the cohort shares are not comparable across periods. The frame path " + "needs exactly one observation per unit-period (a balanced panel)." + ) + # dropna=False: a unit whose label is NaN in one period must fail the + # invariance check, not slip through because nunique() skipped the NaN. + per_unit = data.groupby(unit, sort=True)[first_treat].nunique(dropna=False) + if (per_unit > 1).any(): + offenders = per_unit[per_unit > 1].index.tolist()[:5] + raise ValueError( + f"{first_treat!r} varies within unit(s) {offenders!r}; cohort " + "membership must be time-invariant" + ) + firsts = data.groupby(unit, sort=True)[first_treat].first() + cohorts = firsts.to_numpy() + _validate_cohort_labels(cohorts, unit_ids=firsts.index.to_numpy(), what=first_treat) + periods = np.asarray(sorted(pd.unique(key))) + return cohorts, periods, None + + +def _resolve_cs_inputs( + results: "CallawaySantAnnaResults", +) -> Tuple[np.ndarray, Optional[np.ndarray], int, bool]: + """Read cohort labels (and survey weights) off a fitted CS result. + + The aggregation kit is package-internal, but it is the same channel + ``CallawaySantAnnaResults._aggregate_compute`` already uses - so this is + an established in-package coupling rather than a new one. When the kit is + absent (an old pickle), the caller is pointed at the ``data=`` fallback. + """ + kit = getattr(results, "_aggregation_kit", None) + if kit is None: + raise ValueError( + "this CallawaySantAnnaResults carries no aggregation bookkeeping " + "(it may have been unpickled from an older version), so cohort " + "shares cannot be recovered from it. Pass the panel explicitly:\n" + " attgt_weights(result.to_dataframe('group_time'), data=panel,\n" + " unit=..., time=..., first_treat=...)" + ) + bookkeeping = getattr(kit, "bookkeeping", {}) or {} + cohorts = bookkeeping.get("unit_cohorts") + if cohorts is None: + raise ValueError( + "aggregation bookkeeping does not carry 'unit_cohorts'; pass the " + "panel explicitly via data=/unit=/time=/first_treat=" + ) + weights = bookkeeping.get("survey_weights") + anticipation = int(getattr(kit, "anticipation", 0) or 0) + # Default True: DMLDiD builds its kit through the same builder but its + # precomputed mapping predates the balance flag, and a false rejection + # would be worse than the (already-CS-only) check. + is_balanced = bool(bookkeeping.get("is_balanced", True)) + return ( + np.asarray(cohorts), + (None if weights is None else np.asarray(weights, dtype=float)), + anticipation, + is_balanced, + ) + + +def _guard_cs_design(results: "CallawaySantAnnaResults", aggregation: str) -> None: + """Reject fits whose design breaks the weight formulas. + + These are hard errors rather than warnings: a silently wrong weight table + is worse than no weight table, and every one of these has a concrete fix. + """ + from diff_diff.staggered_results import CallawaySantAnnaResults + + if not isinstance(results, CallawaySantAnnaResults): + raise TypeError( + "attgt_weights takes a CallawaySantAnna (or DMLDiD) fitted result, or an " + f"ATT(g,t) DataFrame; got {type(results).__name__}" + ) + if not getattr(results, "panel", True): + raise ValueError( + "attgt_weights requires a panel fit: E_t[D] and the cohort shares " + "average over a fixed set of units, which repeated cross-sections " + "do not provide. Refit with panel=True." + ) + if getattr(results, "used_rc_on_unbalanced_panel", False): + raise ValueError( + "this fit fell back to repeated-cross-section estimation on an " + "unbalanced panel, so the cohort shares are not comparable across " + "periods. Balance the panel (diff_diff.balance_panel) and refit." + ) + if aggregation != "twfe": + return + control_group = getattr(results, "control_group", None) + if control_group not in (None, "never_treated"): + raise ValueError( + f"aggregation='twfe' requires control_group='never_treated', got " + f"{control_group!r}. The TWFE weight formula is derived against a " + "never-treated comparison group (matching R's twfe_weights, which " + "raises the same restriction)." + ) + base_period = getattr(results, "base_period", None) + if base_period not in (None, "universal"): + raise ValueError( + f"aggregation='twfe' requires base_period='universal', got " + f"{base_period!r}. The formula needs the complete cohort x period " + "grid, including the pre-treatment cells that a varying base does " + "not report. Refit with base_period='universal'." + ) + # R's third restriction: xformla == ~1. The fit records its covariate + # column names on the aggregation kit; a kit without the key predates that + # bookkeeping (an old pickle) and can only be warned about. A missing kit + # is left to _resolve_cs_inputs, whose error is the useful one. + kit = getattr(results, "_aggregation_kit", None) + if kit is None: + return + bookkeeping = getattr(kit, "bookkeeping", {}) or {} + if "covariates" not in bookkeeping: + warnings.warn( + "this fit predates covariate bookkeeping, so attgt_weights cannot " + "verify it used no covariates; the TWFE weight formula assumes an " + "unadjusted regression (R twfe_weights requires xformla == ~1)", + UserWarning, + stacklevel=3, + ) + elif bookkeeping["covariates"]: + raise ValueError( + f"aggregation='twfe' requires a fit without covariates, but this one " + f"adjusted for {list(bookkeeping['covariates'])!r}. The TWFE weight " + "formula describes the unadjusted regression (R's twfe_weights stops " + "unless xformla == ~1); refit with covariates=None, or use " + "decompose_twfe_weights(covariates=...) for the covariate-adjusted " + "decomposition." + ) + + +def attgt_weights( + results: Union["CallawaySantAnnaResults", pd.DataFrame], + *, + aggregation: str = "twfe", + data: Optional[pd.DataFrame] = None, + unit: Optional[str] = None, + time: Optional[str] = None, + first_treat: Optional[str] = None, + weights: Optional[Union[str, np.ndarray]] = None, + anticipation: Optional[int] = None, +) -> ATTGTWeightsResult: + """Weights an estimand places on each group-time effect ATT(g, t). + + Three estimands are available. ``"twfe"`` gives the weights implied by a + static two-way fixed effects regression - the ones that can go negative. + ``"overall"`` and ``"simple"`` give the weights of the Callaway & + Sant'Anna (2021) target parameters ATT^O and ATT^simple, which are + non-negative by construction. Comparing them shows how far the regression + is from the estimand you meant to report. + + Parameters + ---------- + results : CallawaySantAnnaResults or pd.DataFrame + A fitted Callaway & Sant'Anna result (preferred), or a frame with + ``group`` / ``time`` / ``effect`` (or ``att``) columns - the output of + ``result.to_dataframe("group_time")`` is consumed verbatim, including + its ``skip_reason`` column. On the frame path, ``data``, ``unit``, + ``time`` and ``first_treat`` are required so cohort shares can be + formed, and the caller is responsible for the fit having used no + covariates under ``aggregation="twfe"`` (a frame carries no record of + that; the fitted path checks it). + aggregation : {"twfe", "overall", "simple"}, default "twfe" + Which estimand's weights to report. + data : pd.DataFrame, optional + Balanced panel backing the ATT(g, t) frame. Only for the fallback + path; passing it alongside a fitted result raises. + unit, time, first_treat : str, optional + Column names in ``data``. Required together with ``data``. + weights : str or array-like, optional + Unit-level sampling weights (R's ``w=``): a column name in ``data``, + or one value per unit. Rejected when the fit already carries survey + weights, which take precedence. Must be finite and non-negative with + positive treated mass (and positive never-treated mass for ``"twfe"``). + anticipation : int, optional + Anticipation window for the CS estimands, i.e. the number of periods + before ``g`` whose cells count as post-treatment (``t >= g - + anticipation``). Only meaningful on the DataFrame path, where a bare + frame carries no record of the source fit's setting; the fitted path + reads it off the fit and rejects an explicit ``anticipation=``. It does + NOT affect ``aggregation="twfe"``: the TWFE regression's own treatment + indicator is ``1[t >= g]`` regardless of how the CS estimands treat the + run-up, and R's ``twfe_weights`` has no anticipation argument either. + + Returns + ------- + ATTGTWeightsResult + Per-cell weights plus the negative-weight roll-ups. + + Raises + ------ + ValueError + On an unknown ``aggregation``; on a design the formula does not + support (repeated cross-sections, an unbalanced panel, and - for + ``aggregation="twfe"`` - a non-never-treated control group, a + non-universal base period, or a covariate-adjusted fit); on NaN / + ``-inf`` cohort labels, invalid weights, duplicated or non-finite + cells; or on an INCOMPLETE grid: ``"twfe"`` needs every cohort x period + cell, ``"overall"`` / ``"simple"`` every post-treatment cell. + TypeError + When ``results`` is neither a CallawaySantAnna-family result nor a + DataFrame. + + Notes + ----- + Two structural gaps are handled rather than raised, mirroring R: + + * A cohort with NO estimable post-treatment cell (typically one treated in + the first observed period, which has no base period) is dropped from the + table AND from the cohort masses with a warning - what + ``did::pre_process_did`` does when it drops units already treated in the + first period. This drop is allowed ONLY when the cohort is treated in the + first observed period, or when every one of its missing post cells + carries ``skip_reason="zero_treated_control"``; a mid cohort blanked out + by some other mechanism raises rather than disappearing. + * Under ``control_group="not_yet_treated"`` the last cohorts run out of + comparison units, and CS marks those post cells ``zero_treated_control``. + For ``"overall"`` / ``"simple"`` they are treated as structurally absent: + ``"overall"`` divides each cohort by its number of AVAILABLE post periods + and ``"simple"`` renormalizes over the available post cells - what + R ``aggte()`` computes on such a fit. A warning names the cells. The + carve-out keys on the ``skip_reason`` values themselves (that reason is + only ever emitted on a not-yet-treated fit), so the fitted and frame + paths behave identically. (``"twfe"`` requires a never-treated control + group and never reaches this branch.) + + Cohort shares assume a BALANCED panel - the same units observed in every + period - so an unbalanced fitted result is rejected (as + :func:`decompose_twfe_weights` already rejects an unbalanced panel). + + R's ``keep_untreated=TRUE`` is not exposed. It synthesizes ``G = 0`` rows + with ``attgt = 0`` to mirror an internal vector layout; those rows are + excluded from every normalization and contribute exactly zero, so the + argument does not affect any number. + + Examples + -------- + >>> import diff_diff # doctest: +SKIP + >>> cs = diff_diff.CallawaySantAnna(base_period="universal") # doctest: +SKIP + >>> res = cs.fit(df, outcome="y", unit="id", time="t", + ... first_treat="g") # doctest: +SKIP + >>> w = diff_diff.attgt_weights(res, aggregation="twfe") # doctest: +SKIP + >>> print(w.summary()) # doctest: +SKIP + """ + if aggregation not in _AGGREGATIONS: + raise ValueError( + f"aggregation must be one of {list(_AGGREGATIONS)!r}, got " f"{aggregation!r}" + ) + if anticipation is not None: + if isinstance(anticipation, bool) or not isinstance(anticipation, (int, np.integer)): + raise ValueError(f"anticipation must be a non-negative integer, got {anticipation!r}") + if int(anticipation) < 0: + raise ValueError(f"anticipation must be non-negative, got {anticipation!r}") + + frame_path = isinstance(results, pd.DataFrame) + frame = results if isinstance(results, pd.DataFrame) else None + fallback_args = {"data": data, "unit": unit, "time": time, "first_treat": first_treat} + supplied = {k: v for k, v in fallback_args.items() if v is not None} + + if frame_path: + if len(supplied) != 4: + missing = sorted(set(fallback_args) - set(supplied)) + raise ValueError( + "the DataFrame path needs the panel too, so cohort shares can " + f"be formed; missing {missing!r}. Call it as:\n" + " attgt_weights(gt_frame, data=panel, unit='id', " + "time='t', first_treat='g')" + ) + assert data is not None and unit is not None + assert time is not None and first_treat is not None + assert frame is not None + table, skipped = _attgt_from_frame(frame) + cohorts, periods, _ = _unit_cohorts_from_frame(data, unit, time, first_treat) + unit_weights = _resolve_frame_weights(weights, data, unit) + source = "DataFrame" + control_group = None + base_period = None + has_skip_reasons = "skip_reason" in frame.columns + window = 0 if anticipation is None else int(anticipation) + else: + if supplied: + raise ValueError( + f"{sorted(supplied)!r} are only for the DataFrame fallback. A " + "fitted CallawaySantAnnaResults already carries the cohort " + "bookkeeping - drop them, or pass " + "result.to_dataframe('group_time') as the first argument." + ) + if anticipation is not None: + raise ValueError( + "anticipation= is only for the DataFrame fallback; a fitted " + "CallawaySantAnnaResults already carries its own anticipation. " + "Drop anticipation=, or pass result.to_dataframe('group_time') " + "as the first argument." + ) + _guard_cs_design(results, aggregation) + table, skipped = _attgt_from_cs(results) + cohorts, survey_weights, window, is_balanced = _resolve_cs_inputs(results) + if not is_balanced: + raise ValueError( + "attgt_weights requires a balanced panel: the cohort shares and " + "E_t[D] assume the same units are observed in every period. " + "Balance the panel (diff_diff.balance_panel) and refit." + ) + if survey_weights is not None and weights is not None: + raise ValueError( + "this fit already carries survey weights; passing weights= as " + "well is ambiguous. Drop weights= to use the fit's own." + ) + if weights is not None and not isinstance(weights, str): + unit_weights = np.asarray(weights, dtype=float) + elif isinstance(weights, str): + raise ValueError( + "weights= may only name a column on the DataFrame path; pass " + "an array of per-unit weights instead" + ) + else: + unit_weights = survey_weights + periods = np.asarray(results.time_periods) + source = "CallawaySantAnnaResults" + control_group = getattr(results, "control_group", None) + base_period = getattr(results, "base_period", None) + has_skip_reasons = True + + _validate_cohort_labels(cohorts, what="first_treat") + grid = _positional_grid(periods) + n_periods = len(grid) + first_period_pos = 1 + + # Positional mapping FIRST: the cohort universe the masses are formed over + # must be known before the masses are formed. + unit_g_pos = _to_positional_cohort(cohorts, grid) + if not (unit_g_pos != 0).any(): + raise ValueError( + "no ever-treated units found; cohort labels are all never-treated " + "sentinels (0 or inf)" + ) + + # A wrong-length weights= must fail HERE, before the excluded-cohort + # boolean slice below, which would otherwise raise a raw IndexError. + if unit_weights is not None and len(np.asarray(unit_weights)) != len(unit_g_pos): + raise ValueError( + f"weights has length {len(np.asarray(unit_weights))} but the panel " + f"has {len(unit_g_pos)} units" + ) + + g_pos = _to_positional_cohort(table["group"].to_numpy(), grid) + t_pos = np.array([grid[float(t)] for t in table["time"].to_numpy()]) + + # Post-treatment mask. The TWFE regression's own indicator is 1[t >= g] + # regardless of the CS anticipation window; the CS target estimands shift + # it to 1[t >= g - anticipation]. + if aggregation == "twfe": + post_mask = t_pos >= g_pos + else: + post_mask = t_pos >= (g_pos - window) + + def _post_start(g: int) -> int: + raw = g if aggregation == "twfe" else g - window + return max(1, raw) + + # --- whole-cohort exclusion (R did drops units treated in the first period) + present = set(zip(g_pos.tolist(), t_pos.tolist())) + panel_cohorts = sorted({int(g) for g in unit_g_pos if g != 0}) + cohorts_with_post = {int(g) for g in g_pos[post_mask]} + excluded = [g for g in panel_cohorts if g not in cohorts_with_post] + if excluded: + # A cohort may be dropped ONLY when the drop is structural: it is + # treated in the first observed period (R did's first-period drop), or + # every one of its missing post cells carries zero_treated_control + # (it ran out of comparison units). Any other blanked-out cohort must + # fail closed rather than disappear. + structural: List[int] = [] + for g in excluded: + if g == first_period_pos: + structural.append(g) + continue + if has_skip_reasons: + missing_post = [ + (_label_for(grid, g), _label_for(grid, t)) + for t in range(_post_start(g), n_periods + 1) + if (g, t) not in present + ] + if missing_post and all( + skipped.get(lab) == "zero_treated_control" for lab in missing_post + ): + structural.append(g) + not_structural = [g for g in excluded if g not in structural] + if not_structural: + labels = [_label_for(grid, g) for g in not_structural] + if not has_skip_reasons: + raise ValueError( + f"cohort(s) {labels!r} are present in data= but have no " + "post-treatment cell in the ATT(g,t) frame. A bare frame " + "cannot say why; pass result.to_dataframe('group_time') " + "verbatim (it carries skip_reason) or the fitted result itself." + ) + raise ValueError( + f"cohort(s) {labels!r} are present in data= but have no estimable " + "post-treatment cell, and their missing post cell(s) do not all " + "carry skip_reason 'zero_treated_control'. A cohort is only dropped " + "like R did's first-period cohort, or when it runs out of comparison " + "units; blanking a mid cohort's effects is not one of those, so it " + "fails closed instead of silently leaving the estimand." + ) + n_units_excl = int(np.isin(unit_g_pos, excluded).sum()) + warnings.warn( + f"cohort(s) {[_label_for(grid, g) for g in excluded]!r} ({n_units_excl} " + "unit(s)) have no estimable post-treatment cell and were dropped from " + "the weight table and the cohort shares: either treated in the first " + "observed period (R did's first-period drop) or run out of comparison " + "units under a not-yet-treated control group (R did's panel truncation)", + UserWarning, + stacklevel=2, + ) + keep_units = ~np.isin(unit_g_pos, excluded) + cohorts = cohorts[keep_units] + unit_g_pos = unit_g_pos[keep_units] + if unit_weights is not None: + unit_weights = np.asarray(unit_weights, dtype=float)[keep_units] + keep_rows = ~np.isin(g_pos, excluded) + table = table[keep_rows].reset_index(drop=True) + g_pos, t_pos, post_mask = g_pos[keep_rows], t_pos[keep_rows], post_mask[keep_rows] + skipped = {k: v for k, v in skipped.items() if _pos_of(grid, k[0]) not in excluded} + + p_all, p_treated, e_dt, mean_e_dt = _cohort_masses( + cohorts, grid, unit_weights, require_control_mass=(aggregation == "twfe") + ) + + # --- grid completeness + present = set(zip(g_pos.tolist(), t_pos.tolist())) + surviving = sorted(cohorts_with_post) + if aggregation == "twfe": + required = {(g, t) for g in surviving for t in range(1, n_periods + 1)} + else: + required = {(g, t) for g in surviving for t in range(_post_start(g), n_periods + 1)} + missing_cells = sorted(required - present) + structurally_absent: List[Tuple[Any, Any]] = [] + if missing_cells: + # The zero_treated_control carve-out is keyed on the skip_reason VALUE, + # not on control_group: that reason is only ever emitted on a + # not-yet-treated fit, and the frame path has no control_group to read, + # so keying on the reason is what makes the two paths agree. + carve_out_ok = aggregation != "twfe" + hard: List[Tuple[Tuple[Any, Any], Optional[str]]] = [] + for g, t in missing_cells: + label = (_label_for(grid, g), _label_for(grid, t)) + reason = skipped.get(label) + if carve_out_ok and reason == "zero_treated_control": + structurally_absent.append(label) + else: + hard.append((label, reason)) + if hard: + what = "cohort x period" if aggregation == "twfe" else "post-treatment" + detail = ", ".join( + f"{lab} [{reason or 'not in source table'}]" for lab, reason in hard[:6] + ) + raise ValueError( + f"aggregation={aggregation!r} needs the complete {what} grid, but " + f"{len(hard)} required cell(s) are missing: {detail}. A weight table " + "over a partial grid is not the named estimand. Fix the source fit " + "(or pass the complete to_dataframe('group_time') output)." + ) + warnings.warn( + f"{len(structurally_absent)} post-treatment cell(s) {structurally_absent[:6]!r} " + "have no not-yet-treated comparison units (skip_reason " + "'zero_treated_control') and are treated as structurally absent: " + f"aggregation={aggregation!r} averages over each cohort's AVAILABLE " + "post periods, as R aggte() does on a not-yet-treated fit", + UserWarning, + stacklevel=2, + ) + + # Non-estimable PRE cells of surviving cohorts are the only drops left; + # the CS estimands ignore pre cells, so they change nothing. + dropped = 0 + surviving_positional = {int(g) for g in cohorts_with_post} + for g_lab, t_lab in skipped: + gp, tp = _pos_of(grid, g_lab), _pos_of(grid, t_lab) + if gp in surviving_positional and tp < _post_start(gp): + dropped += 1 + if dropped and aggregation != "twfe": + warnings.warn( + f"{dropped} pre-treatment group-time cell(s) had no estimable ATT(g,t) " + f"and were excluded; aggregation={aggregation!r} places no weight on " + "pre-treatment cells, so the weights are unaffected", + UserWarning, + stacklevel=2, + ) + + if aggregation == "twfe": + weight_vec = _twfe_weight_vector(g_pos, t_pos, n_periods, p_all, e_dt, mean_e_dt) + elif aggregation == "overall": + n_post_available = {g: int(((g_pos == g) & post_mask).sum()) for g in surviving} + weight_vec = _overall_weight_vector(g_pos, p_treated, post_mask, n_post_available) + else: + weight_vec = _simple_weight_vector(g_pos, p_treated, post_mask) + + out = pd.DataFrame( + { + "group": table["group"].to_numpy(), + "time": table["time"].to_numpy(), + "post": post_mask.astype(int), + "weight": weight_vec, + "att": table["att"].to_numpy(), + } + ) + + negative = weight_vec < 0 + abs_total = float(np.abs(weight_vec).sum()) + negative_post = negative & post_mask + abs_post_total = float(np.abs(weight_vec[post_mask]).sum()) + return ATTGTWeightsResult( + weights=out, + aggregation=aggregation, + implied_att=float((weight_vec * table["att"].to_numpy()).sum()), + n_negative=int(negative.sum()), + negative_weight_share=( + float(np.abs(weight_vec[negative]).sum() / abs_total) if abs_total > 0 else 0.0 + ), + n_negative_post=int(negative_post.sum()), + negative_post_weight_share=( + float(np.abs(weight_vec[negative_post]).sum() / abs_post_total) + if abs_post_total > 0 + else 0.0 + ), + n_cells=len(out), + source=source, + control_group=control_group, + base_period=base_period, + n_dropped_cells=dropped, + ) + + +def _label_for(grid: Dict[float, int], pos: int) -> Any: + """Positional period -> original label (inverse of ``_positional_grid``).""" + for label, p in grid.items(): + if p == pos: + return int(label) if float(label).is_integer() else label + return pos + + +def _pos_of(grid: Dict[float, int], label: Any) -> int: + """Original label -> positional period; never-treated sentinel stays 0.""" + try: + value = float(label) + except (TypeError, ValueError): + return -1 + if value == 0 or value == np.inf: + return 0 + return grid.get(value, -1) + + +def _resolve_frame_weights( + weights: Optional[Union[str, np.ndarray]], + data: pd.DataFrame, + unit: str, +) -> Optional[np.ndarray]: + """Turn ``weights=`` into one value per unit, or None.""" + if weights is None: + return None + if isinstance(weights, str): + if weights not in data.columns: + raise ValueError(f"weights column {weights!r} not found in data") + per_unit = data.groupby(unit, sort=True)[weights].nunique(dropna=False) + if (per_unit > 1).any(): + offenders = per_unit[per_unit > 1].index.tolist()[:5] + raise ValueError( + f"weights column {weights!r} varies within unit(s) " + f"{offenders!r}; sampling weights must be time-invariant" + ) + return data.groupby(unit, sort=True)[weights].first().to_numpy(dtype=float) + return np.asarray(weights, dtype=float) + + +# --------------------------------------------------------------------------- +# Panel plumbing for the decomposition +# --------------------------------------------------------------------------- + + +def _weighted_mean(values: np.ndarray, weights: np.ndarray) -> float: + """``stats::weighted.mean`` on flat arrays.""" + total = weights.sum() + if total == 0: + return float("nan") + return float((values * weights).sum() / total) + + +def _effective_sample_size(est_weights: np.ndarray, sampling_weights: np.ndarray) -> float: + """``sum(w)^2 / sum(w^2)`` after normalizing both weight vectors.""" + sw = sampling_weights / sampling_weights.mean() + ew = est_weights / _weighted_mean(est_weights, sw) + denom = float((ew**2).sum()) + if denom == 0: + return float("nan") + return float(ew.sum() ** 2 / denom) + + +def _require_finite(block: np.ndarray, name: str, *, what: str) -> np.ndarray: + """Fail closed on NaN / inf in an estimation input block. + + A NaN outcome otherwise propagates silently: the demeaned residual becomes + NaN and every ``(g, t)`` ATT(g, t) is NaN, so the decomposition returns an + all-NaN result with no error. Complete-case handling is a policy choice we + do not make here, so the caller must clean the input. + """ + if not np.all(np.isfinite(block)): + n_bad = int((~np.isfinite(block)).sum()) + raise ValueError( + f"{what} {name!r} contains {n_bad} non-finite value(s) (NaN or inf); " + "decompose_twfe_weights does not drop incomplete cases, so clean the " + f"panel first (e.g. drop or impute rows with a missing {what})" + ) + return block + + +class _Panel: + """Balanced panel reshaped to ``(n_units, n_periods)`` with positional time. + + Sorting by ``(unit, period)`` and reshaping means every ``(g, t)`` slice + is a plain boolean row mask plus a column index, instead of repeated + boolean scans over the long frame. + """ + + def __init__( + self, + data: pd.DataFrame, + *, + outcome: str, + unit: str, + time: str, + first_treat: str, + covariates: Sequence[str], + weights: Optional[str], + ) -> None: + for col in (outcome, unit, time, first_treat, *covariates): + if col not in data.columns: + raise ValueError(f"column {col!r} not found in data") + if weights is not None and weights not in data.columns: + raise ValueError(f"weights column {weights!r} not found in data") + + time_key = _validate_time_labels(data[time].to_numpy(), what=time) + frame = data.assign(_twfe_time_key=time_key) + frame = frame.sort_values([unit, "_twfe_time_key"]).reset_index(drop=True) + units = frame[unit].to_numpy() + periods = frame["_twfe_time_key"].to_numpy(dtype=float) + self.unit_ids = np.asarray(sorted(pd.unique(units))) + self.period_labels = np.asarray(sorted(pd.unique(periods))) + n_units = len(self.unit_ids) + n_periods = len(self.period_labels) + if len(frame) != n_units * n_periods: + raise ValueError( + f"decompose_twfe_weights requires a balanced panel: got " + f"{len(frame)} rows for {n_units} units x {n_periods} periods. " + "Balance it first, e.g. diff_diff.balance_panel(data, unit=..., " + "time=...)." + ) + counts = frame.groupby(unit, sort=True)["_twfe_time_key"].nunique().to_numpy() + if not np.all(counts == n_periods): + raise ValueError( + "decompose_twfe_weights requires a balanced panel: some units " + "are missing periods" + ) + + self.grid = _positional_grid(self.period_labels) + self.n_units = n_units + self.n_periods = n_periods + + cohort_long = frame[first_treat].to_numpy() + # dropna=False: a NaN label in one period must fail invariance, not + # be skipped by nunique(). + per_unit = frame.groupby(unit, sort=True)[first_treat].nunique(dropna=False) + if (per_unit > 1).any(): + offenders = per_unit[per_unit > 1].index.tolist()[:5] + raise ValueError( + f"{first_treat!r} varies within unit(s) {offenders!r}; cohort " + "membership must be time-invariant" + ) + raw_cohorts = cohort_long.reshape(n_units, n_periods)[:, 0] + _validate_cohort_labels(raw_cohorts, unit_ids=self.unit_ids, what=first_treat) + self.cohorts = _to_positional_cohort(raw_cohorts, self.grid) + if not (self.cohorts == 0).any(): + raise ValueError( + "decompose_twfe_weights needs never-treated units as the " + "comparison group; none were found (matching R's twfeweights, " + "which supports only a never-treated comparison)" + ) + self.outcome = _require_finite( + frame[outcome].to_numpy(dtype=float).reshape(n_units, n_periods), + outcome, + what="outcome", + ) + if weights is None: + self.weights = np.ones((n_units, n_periods)) + else: + block = frame[weights].to_numpy(dtype=float).reshape(n_units, n_periods) + # Finite check FIRST: np.allclose is False on any NaN, which would + # otherwise be misreported as "varies within unit". + if not np.all(np.isfinite(block)): + raise ValueError( + f"weights column {weights!r} must be finite; got NaN or infinite weight(s)" + ) + if not np.allclose(block, block[:, :1]): + raise ValueError( + f"weights column {weights!r} varies within unit; sampling " + "weights must be time-invariant" + ) + _validate_unit_weights(block[:, 0], self.cohorts == 0, require_control_mass=True) + self.weights = block + self.covariates = tuple(covariates) + if covariates: + self.design = _require_finite( + frame[list(covariates)] + .to_numpy(dtype=float) + .reshape(n_units, n_periods, len(covariates)), + ", ".join(covariates), + what="covariate", + ) + else: + self.design = np.zeros((n_units, n_periods, 0)) + + periods_positional = np.arange(1, n_periods + 1) + self.treated = ( + (periods_positional[None, :] >= self.cohorts[:, None]) & (self.cohorts[:, None] != 0) + ).astype(float) + + # Two-way demeaning through the house helper (the same alternating + # projections fixest::demean runs), on the sorted long frame so the + # (unit, period) reshape afterwards is a plain view. The treatment + # indicator is DERIVED from cohorts x positional periods, not an input + # column, so it is synthesized here before the call. Both the RAW and + # the demeaned covariate blocks are kept: the annihilation filter in + # _fwl_residuals compares one against the other. + demean_frame = pd.DataFrame( + {"_unit": frame[unit].to_numpy(), "_time": frame["_twfe_time_key"].to_numpy()} + ) + demean_frame["_treated"] = self.treated.reshape(-1) + for j, name in enumerate(self.covariates): + demean_frame[f"_x{j}"] = self.design[:, :, j].reshape(-1) + row_weights = None if weights is None else self.weights.reshape(-1) + demeaned = within_transform( + demean_frame, + ["_treated", *(f"_x{j}" for j in range(len(self.covariates)))], + "_unit", + "_time", + weights=row_weights, + suffix="_dm", + tol=1e-12, + ) + self.treated_demeaned = ( + demeaned["_treated_dm"].to_numpy(dtype=float).reshape(n_units, n_periods) + ) + if self.covariates: + self.design_demeaned = np.stack( + [ + demeaned[f"_x{j}_dm"].to_numpy(dtype=float).reshape(n_units, n_periods) + for j in range(len(self.covariates)) + ], + axis=2, + ) + else: + self.design_demeaned = np.zeros((n_units, n_periods, 0)) + + def covariate_block( + self, names: Sequence[str], data: pd.DataFrame, unit: str, time: str + ) -> np.ndarray: + """Unit-mean-collapsed covariates, one column per name. + + R's ``twfe_cov_bal`` averages each balance covariate over ALL periods + within a unit before comparing groups, so a time-varying covariate is + summarized by its unit mean. + """ + frame = data.assign(_twfe_time_key=_validate_time_labels(data[time].to_numpy(), what=time)) + frame = frame.sort_values([unit, "_twfe_time_key"]).reset_index(drop=True) + block = ( + frame[list(names)] + .to_numpy(dtype=float) + .reshape(self.n_units, self.n_periods, len(names)) + ) + for j, name in enumerate(names): + _require_finite(block[:, :, j], name, what="balance covariate") + return block.mean(axis=1) + + +def _fwl_residuals(panel: _Panel) -> Tuple[np.ndarray, float, List[str]]: + """Frisch-Waugh-Lovell residual of treatment on covariates, plus its scale. + + Double-demeans ``D`` and ``X``, projects the demeaned treatment on the + demeaned covariates, and returns the residual. That residual IS the + implicit weight the regression applies to each observation; ``alpha_den`` + is the normalization ``E[resid * Ddot]`` from R's + ``combine_twfe_weights_gt``. The third element is the names of the + covariates that SURVIVED the annihilation filter, so the result reports the + columns the regression actually used rather than the user's input list. + + With no covariates the projection is empty and the residual is just the + double-demeaned treatment - which is exactly the branch R cannot run, + because ``fixest::demean`` segfaults on the zero-column model matrix it + builds for ``xformula = ~1``. + """ + weights = panel.weights + d_dot = panel.treated_demeaned + x_dot = panel.design_demeaned + + flat_d = d_dot.reshape(-1) + flat_w = weights.reshape(-1) + # Explicit row count: with zero covariates the trailing axis is 0 and + # numpy cannot infer a -1 against it. This is the same no-covariate branch + # on which fixest::demean segfaults; here it simply has to be spelled out. + n_obs = panel.n_units * panel.n_periods + flat_x = x_dot.reshape(n_obs, x_dot.shape[2]) + + # Numerical hygiene: drop covariates that double-demeaning ANNIHILATED + # before anything is projected on them. A time-invariant regressor leaves a + # column of pure rounding noise (~1e-16 against a raw scale of ~1). Keeping + # it is not catastrophic - the column lies in the FE span and is orthogonal + # to the treatment residual, so on mpdta's `lpop` it moves the FWL residual + # by ~2e-18 - but regressing on an exactly-zero column is meaningless, and + # dropping it is what makes covariates=None and covariates=[] + # agree exactly. The test is scale-relative: a column counts as having no + # within-variation when its demeaned norm is negligible NEXT TO ITS OWN raw + # norm, which a rank test on the demeaned matrix alone cannot see (there, + # 1e-16 is simply the largest pivot). + # + # The threshold is the accumulated ROUNDING-NOISE scale, not a fixed + # relative constant: demeaning an ``n_obs``-row column accumulates + # ``O(sqrt(n) * eps)`` of relative error, times a safety factor of 64. + # A fixed 1e-10 was five orders too loose - it discarded a covariate with + # level 1e6 and genuine within-sd 1e-4 (ratio 1e-10) - while still + # annihilating mpdta's `lpop`, whose true within-variation is many orders + # above the noise floor. + raw_scale = np.linalg.norm( + panel.design.reshape(n_obs, x_dot.shape[2]), + axis=0, + ) + demeaned_scale = np.linalg.norm(flat_x, axis=0) + noise_floor = np.sqrt(n_obs) * 64.0 * np.finfo(float).eps + annihilated = demeaned_scale <= noise_floor * np.maximum(raw_scale, 1.0) + if annihilated.any(): + names = [panel.covariates[j] for j in np.flatnonzero(annihilated)] + warnings.warn( + f"covariate(s) {names!r} have no within-unit-and-period variation " + "(or within-variation at the floating-point noise floor of their own " + "level) and were dropped: two-way demeaning annihilates them, so they " + "cannot affect a two-way fixed effects regression. If that is not " + "intended, centre or rescale the covariate so its within-variation is " + "not negligible next to its level", + UserWarning, + stacklevel=3, + ) + flat_x = flat_x[:, ~annihilated] + surviving = [name for name, drop in zip(panel.covariates, annihilated) if not drop] + + if flat_x.shape[1]: + # House solver: WLS through the origin (R's lm(y ~ -1 + X, w)). On a + # rank-deficient design it fits the maximal independent set, sets the + # dropped coefficients to NaN (R-style) and computes the residual from + # the identified ones - so the residual is the FWL residual we need + # and the NaN positions name the collinear columns. + gamma, resid, _ = solve_ols( + flat_x, + flat_d, + weights=flat_w, + return_vcov=False, + rank_deficient_action="silent", + column_names=list(surviving), + ) + dropped = np.flatnonzero(np.isnan(gamma)) + if dropped.size: + names = [surviving[j] for j in dropped] + warnings.warn( + f"dropped collinear covariate column(s) {names!r} after " + "double-demeaning; they carry no within-variation independent of " + "the others", + UserWarning, + stacklevel=3, + ) + resid = np.asarray(resid, dtype=float) + else: + resid = flat_d + alpha_den = _weighted_mean(resid * flat_d, flat_w) + if not np.isfinite(alpha_den) or alpha_den == 0: + raise ValueError( + "the treatment indicator has no within-variation left after " + "double-demeaning and covariate adjustment, so the TWFE " + "coefficient is not identified" + ) + return resid.reshape(panel.n_units, panel.n_periods), alpha_den, surviving + + +def _normalize_cell_weights( + resid: np.ndarray, sampling_weights: np.ndarray, scale: float +) -> Tuple[np.ndarray, bool]: + """Scale a cell's residuals to mean one, handling the 0/0 case. + + The implicit weights within a cell are ``resid / mean(resid)``. For the + never-treated comparison group the residual is CONSTANT within a period + (their treatment indicator is identically zero, so the double-demeaned + value is ``-E_t[D] + mean_t E_t[D]``, the same for every control unit) - + and for some cohort structures that constant is analytically ZERO. On + sim_staggered (three equal cohorts at g in {0,3,4}, T=5) it vanishes + exactly at t=3: ``-1/3 + 1/3``. + + That makes the ratio 0/0. The limit is unambiguous - a constant divided + by its own mean is one - so return exactly one rather than dividing two + rounding errors. R divides anyway, which is why its per-cell ATT(g,t) at + such a cell carries ~1e-4 of noise; the aggregate is unaffected because + the weights on the affected cells cancel exactly. + + Returns the weights and whether the degenerate branch was taken. + """ + mean = _weighted_mean(resid, sampling_weights) + spread = float(np.max(resid) - np.min(resid)) if resid.size else 0.0 + tol = 1e-12 * max(scale, 1.0) + if abs(mean) <= tol: + if spread <= tol: + return np.ones_like(resid), True + raise ValueError( + "a group-time cell has comparison-group implicit weights that " + "average to zero but are not constant, so the cell's ATT(g,t) is " + "not identified. This usually means the panel has too little " + "variation in treatment timing." + ) + return resid / mean, False + + +def _decompose_fwl( + panel: _Panel, + base_period: str, + balance_covariates: Sequence[str], + balance_block: Optional[np.ndarray], +) -> Dict[str, Any]: + """R ``implicit_twfe_weights``: TWFE as weighted ATT(g, t) + a remainder.""" + resid, alpha_den, surviving_covariates = _fwl_residuals(panel) + weights = panel.weights + flat_w = weights.reshape(-1) + cohorts = panel.cohorts + treated_cohorts = sorted({int(g) for g in cohorts if g != 0}) + if not treated_cohorts: + raise ValueError("no ever-treated units found; nothing to decompose") + control_mask = cohorts == 0 + if not control_mask.any(): + raise ValueError( + "decompose_twfe_weights needs never-treated units as the " + "comparison group; none were found (matching R's twfeweights, " + "which supports only a never-treated comparison)" + ) + if base_period == "gmin1" and 1 in treated_cohorts: + raise ValueError( + "base_period='gmin1' needs a period before each cohort's " + "treatment, but a cohort is treated in the first period. Use " + "base_period='first_period', or drop that cohort." + ) + + resid_scale = float(np.abs(resid).max()) + cells: List[Dict[str, Any]] = [] + balance_rows: List[Dict[str, Any]] = [] + degenerate_cells: List[Tuple[Any, Any]] = [] + for g in treated_cohorts: + treated_mask = cohorts == g + for t_pos in range(1, panel.n_periods + 1): + col = t_pos - 1 + w_treated = weights[treated_mask, col] + w_control = weights[control_mask, col] + + r_treated = resid[treated_mask, col] + r_control = resid[control_mask, col] + gpart_w, _ = _normalize_cell_weights(r_treated, w_treated, resid_scale) + upart_w, degenerate = _normalize_cell_weights(r_control, w_control, resid_scale) + if degenerate: + degenerate_cells.append((panel.period_labels[g - 1], panel.period_labels[col])) + + y_t = panel.outcome[:, col] + if base_period == "first_period": + base = panel.outcome[:, 0] + else: + base = panel.outcome[:, g - 2] + adjusted = y_t - base + + gpart = _weighted_mean(gpart_w * adjusted[treated_mask], w_treated) + upart = _weighted_mean(upart_w * adjusted[control_mask], w_control) + + p_g = _weighted_mean( + (cohorts == g).astype(float)[:, None].repeat(panel.n_periods, axis=1).reshape(-1), + flat_w, + ) + alpha_weight = ( + _weighted_mean(r_treated, w_treated) * p_g / (alpha_den * panel.n_periods) + ) + + remainder = 0.0 + if base_period == "gmin1": + y_gmin1 = panel.outcome[:, g - 2] + remainder = -_weighted_mean(upart_w * y_gmin1[control_mask], w_control) + + cells.append( + { + "group": panel.period_labels[g - 1], + "time": panel.period_labels[col], + "post": int(t_pos >= g), + "att": gpart - upart, + "weight": alpha_weight, + "ess": _effective_sample_size(upart_w, w_control), + "remainder": remainder, + } + ) + if balance_block is not None: + balance_rows.extend( + _balance_cell( + balance_block, + balance_covariates, + treated_mask, + control_mask, + gpart_w, + upart_w, + w_treated, + w_control, + group=panel.period_labels[g - 1], + time=panel.period_labels[col], + post=int(t_pos >= g), + ) + ) + + if degenerate_cells: + warnings.warn( + f"{len(degenerate_cells)} group-time cell(s) {degenerate_cells[:4]!r}" + " have comparison-group implicit weights that are constant and " + "average to zero, so their ATT(g,t) is a 0/0 limit (taken as the " + "unweighted contrast). The weights on these cells cancel in the " + "aggregate, so `estimate` is unaffected; read the individual " + "ATT(g,t) there with caution", + UserWarning, + stacklevel=3, + ) + + frame = pd.DataFrame(cells) + weight_vec = frame["weight"].to_numpy() + att_col = frame["att"].to_numpy() + post_col = frame["post"].to_numpy().astype(bool) + decomposition = float((weight_vec * att_col).sum()) + remainder_total = float((frame["remainder"].to_numpy() * weight_vec).sum()) + ess_col = frame["ess"].to_numpy() + return { + "cells": frame, + "estimate": decomposition + remainder_total, + "decomposition": decomposition, + "remainder": remainder_total, + "pre_period_contribution": float((weight_vec[~post_col] * att_col[~post_col]).sum()), + "post_only": float((weight_vec[post_col] * att_col[post_col]).sum()), + # summary.decomposed_twfe: post cells only, on both factors + "effective_sample_size": float( + post_col.sum() * (weight_vec[post_col] * ess_col[post_col]).sum() + ), + "balance": pd.DataFrame(balance_rows) if balance_block is not None else None, + "covariates": tuple(surviving_covariates), + } + + +# --------------------------------------------------------------------------- +# Balance statistics (Imbens & Rubin 2015, as implemented upstream) +# --------------------------------------------------------------------------- + + +def _weighted_ecdf(values: np.ndarray, weights: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """``BMisc::weighted_ecdf``: knots and CDF heights. + + ``weights`` are normalized by their mean, the knots are the sorted unique + values, and ``F(knot_j) = mean(w * (y <= knot_j))``. + """ + w = weights / weights.mean() + # Sort once and read cumulative mass at each unique-value boundary, rather + # than rescanning the full vector per knot (the naive form is O(n * k), and + # this runs per covariate x cohort x period). `np.unique` returns the + # sorted knots, so a single searchsorted locates each boundary. + order = np.argsort(values, kind="stable") + sorted_values = values[order] + cumulative = np.cumsum(w[order]) + knots = np.unique(values) + last = np.searchsorted(sorted_values, knots, side="right") - 1 + heights = cumulative[last] / len(values) + return knots, heights + + +def _ecdf_eval(knots: np.ndarray, heights: np.ndarray, at: float) -> float: + """Evaluate the step function from ``BMisc::make_dist``. + + ``approxfun(method="constant", yleft=0, yright=1, f=0)``: the value on + ``[knot_i, knot_{i+1})`` is ``heights[i]``, zero below the first knot and + one above the last. + """ + if at < knots[0]: + return 0.0 + if at > knots[-1]: + return 1.0 + idx = int(np.searchsorted(knots, at, side="right") - 1) + return float(heights[idx]) + + +def _ecdf_quantile(knots: np.ndarray, heights: np.ndarray, prob: float) -> float: + """``stats:::quantile.ecdf``: type-7 quantile of a reconstructed sample. + + R does NOT invert the step function directly. It rebuilds a pseudo-sample + by repeating each knot ``diff(c(0, round(nobs * F)))`` times - where + ``nobs`` is the number of KNOTS, not the number of observations - and then + takes an ordinary type-7 quantile of that. Reproduced exactly, because the + rounding makes the result differ from a direct inversion. + """ + nobs = len(knots) + counts = np.diff(np.concatenate([[0.0], np.round(nobs * heights)])) + counts = np.maximum(counts, 0).astype(int) + sample = np.repeat(knots, counts) + if sample.size == 0: + return float("nan") + # R's default type-7 quantile. + sample = np.sort(sample) + h = (len(sample) - 1) * prob + lo = int(np.floor(h)) + hi = min(lo + 1, len(sample) - 1) + return float(sample[lo] + (h - lo) * (sample[hi] - sample[lo])) + + +def _pooled_sd(x: np.ndarray, treated: np.ndarray, sampling_weights: np.ndarray) -> float: + """Pooled standard deviation across the treated and comparison groups.""" + sw = sampling_weights / sampling_weights.mean() + + def wvar(values: np.ndarray, w: np.ndarray) -> float: + return _weighted_mean((values - _weighted_mean(values, w)) ** 2, w) + + var1 = wvar(x[treated == 1], sw[treated == 1]) + var0 = wvar(x[treated == 0], sw[treated == 0]) + n1 = sw[treated == 1].sum() + n0 = sw[treated == 0].sum() + if n1 + n0 - 2 <= 0: + return float("nan") + return float(np.sqrt(((n1 - 1) * var1 + (n0 - 1) * var0) / (n1 + n0 - 2))) + + +def _normalize_est_weights( + est_weights: np.ndarray, treated: np.ndarray, sw: np.ndarray +) -> np.ndarray: + """Scale estimation weights to mean one WITHIN each group, as R does.""" + out = np.array(est_weights, dtype=float, copy=True) + for group in (0, 1): + mask = treated == group + if mask.any(): + out[mask] = out[mask] / _weighted_mean(out[mask], sw[mask]) + return out + + +def _log_ratio_sd( + x: np.ndarray, + treated: np.ndarray, + est_weights: np.ndarray, + sampling_weights: np.ndarray, +) -> float: + """Log ratio of treated to comparison spread. + + Note: upstream scales each group's SD by ``sqrt(n - 1)`` before taking + the ratio, which is not a conventional standard deviation. Preserved + verbatim for parity - the quantity is only ever read as a relative + balance statistic, and the extra factor largely cancels in the ratio. + """ + sw = sampling_weights / sampling_weights.mean() + ew = _normalize_est_weights(est_weights, treated, sw) + + def wvar(values: np.ndarray, e: np.ndarray, w: np.ndarray) -> float: + scaled = values * e + return _weighted_mean((scaled - _weighted_mean(scaled, w)) ** 2, w) + + var1 = wvar(x[treated == 1], ew[treated == 1], sw[treated == 1]) + var0 = wvar(x[treated == 0], ew[treated == 0], sw[treated == 0]) + n1 = sw[treated == 1].sum() + n0 = sw[treated == 0].sum() + sd1 = np.sqrt(max(n1 - 1, 0)) * np.sqrt(var1) + sd0 = np.sqrt(max(n0 - 1, 0)) * np.sqrt(var0) + if sd1 <= 0 or sd0 <= 0: + return float("nan") + return float(np.log(sd1) - np.log(sd0)) + + +def _frac_treated_extreme( + x: np.ndarray, + treated: np.ndarray, + est_weights: np.ndarray, + sampling_weights: np.ndarray, + alpha: float = 0.05, +) -> float: + """Share of treated mass outside the comparison group's central range. + + A step function of a weighted empirical CDF, so a perturbation of order + 1e-12 can move one unit across a knot and shift the value by 1/n. Tests + gate it with an absolute tolerance of ``1 / n_control`` rather than a + relative one. + """ + if len(np.unique(x)) < 3: + return float("nan") + sw = sampling_weights / sampling_weights.mean() + ew = _normalize_est_weights(est_weights, treated, sw) + + control = treated == 0 + treat = treated == 1 + knots_u, heights_u = _weighted_ecdf(ew[control] * x[control], sw[control]) + upper = _ecdf_quantile(knots_u, heights_u, 1 - alpha / 2) + lower = _ecdf_quantile(knots_u, heights_u, alpha / 2) + knots_t, heights_t = _weighted_ecdf(ew[treat] * x[treat], sw[treat]) + return float( + 1.0 - _ecdf_eval(knots_t, heights_t, upper) + _ecdf_eval(knots_t, heights_t, lower) + ) + + +def _balance_cell( + block: np.ndarray, + names: Sequence[str], + treated_mask: np.ndarray, + control_mask: np.ndarray, + weights_treated: np.ndarray, + weights_control: np.ndarray, + sw_treated: np.ndarray, + sw_control: np.ndarray, + *, + group: Any, + time: Any, + post: int, +) -> List[Dict[str, Any]]: + """Per-covariate implicit-weight balance for one ``(g, t)`` cell.""" + both = treated_mask | control_mask + indicator = np.where(treated_mask[both], 1, 0) + est = np.empty(int(both.sum())) + est[indicator == 1] = weights_treated + est[indicator == 0] = weights_control + sw_both = np.empty_like(est) + sw_both[indicator == 1] = sw_treated + sw_both[indicator == 0] = sw_control + ones = np.ones_like(est) + + rows: List[Dict[str, Any]] = [] + for j, name in enumerate(names): + col = block[:, j] + x_t = col[treated_mask] + x_c = col[control_mask] + x_both = col[both] + unweighted_treated = _weighted_mean(x_t, sw_treated) + unweighted_control = _weighted_mean(x_c, sw_control) + weighted_treated = _weighted_mean(x_t * weights_treated, sw_treated) + weighted_control = _weighted_mean(x_c * weights_control, sw_control) + rows.append( + { + "group": group, + "time": time, + "post": post, + "covariate": name, + "unweighted_treated": unweighted_treated, + "unweighted_control": unweighted_control, + "unweighted_diff": unweighted_treated - unweighted_control, + "weighted_treated": weighted_treated, + "weighted_control": weighted_control, + "weighted_diff": weighted_treated - weighted_control, + "sd": _pooled_sd(x_both, indicator, sw_both), + "unweighted_log_ratio_sd": _log_ratio_sd(x_both, indicator, ones, sw_both), + "weighted_log_ratio_sd": _log_ratio_sd(x_both, indicator, est, sw_both), + "unweighted_frac_extreme": _frac_treated_extreme(x_both, indicator, ones, sw_both), + "weighted_frac_extreme": _frac_treated_extreme(x_both, indicator, est, sw_both), + } + ) + return rows + + +_METHODS = ("fwl",) +_BASE_PERIODS = ("first_period", "gmin1") + + +def decompose_twfe_weights( + data: pd.DataFrame, + *, + outcome: str, + unit: str, + time: str, + first_treat: str, + method: str = "fwl", + covariates: Optional[Sequence[str]] = None, + base_period: str = "first_period", + balance_covariates: Optional[Sequence[str]] = None, + weights: Optional[str] = None, +) -> TWFEDecompositionResult: + """Decompose a TWFE estimate into weighted group-time effects. + + Runs the regression, recovers the implicit weight it places on each + ATT(g, t), and separates out the part of the estimate that comes from + PRE-treatment cells (``pre_period_contribution``). That component can + reflect differential pre-trends OR plain sampling variation - the + diagnostic carries no inference - so it is diagnostic evidence about the + earlier-period restrictions, not proof that the identifying assumption + fails in the post-treatment counterfactual. + + Takes the raw panel rather than a fitted result, because it re-estimates: + it double-demeans treatment and covariates and forms its own group-time + contrasts, so there is no ATT(g, t) table it could consume. Its companion + :func:`attgt_weights` is the fitted-result surface, and the two are tied + by an identity that holds when the fit used ``base_period="universal"``, + ``control_group="never_treated"`` and no covariates:: + + sum(attgt_weights(cs, aggregation="twfe").weights.eval("weight * att")) + == decompose_twfe_weights(panel, ...).estimate + + Parameters + ---------- + data : pd.DataFrame + Balanced panel in long form. + outcome, unit, time, first_treat : str + Column names, matching :meth:`CallawaySantAnna.fit`. Never-treated + units carry ``first_treat`` of ``0`` (or ``inf``). + method : {"fwl"}, default "fwl" + ``"fwl"`` recovers the Frisch-Waugh-Lovell implicit weights from the + TWFE regression. + covariates : sequence of str, optional + Covariates the regression adjusts for. ``None`` runs the + no-covariate decomposition. + base_period : {"first_period", "gmin1"}, default "first_period" + Which pre-period each cell is measured against. ``"gmin1"`` (the + period before treatment) generates a non-zero ``remainder``. + balance_covariates : sequence of str, optional + Covariates to report implicit-weight balance for, readable afterwards + via :meth:`TWFEDecompositionResult.covariate_balance`. Each is + averaged over periods within unit before groups are compared, as + upstream does. + weights : str, optional + Time-invariant sampling-weight column. + + Returns + ------- + TWFEDecompositionResult + + Raises + ------ + ValueError + On an unknown ``method`` or ``base_period``; on an unbalanced panel, + a missing never-treated group, or time-varying cohort labels; or when + the treatment has no within-variation left after demeaning. + + Examples + -------- + >>> import diff_diff # doctest: +SKIP + >>> dec = diff_diff.decompose_twfe_weights( # doctest: +SKIP + ... panel, outcome="y", unit="id", time="t", first_treat="g", + ... covariates=["x"], balance_covariates=["x"], + ... ) + >>> dec.pre_period_contribution # doctest: +SKIP + >>> dec.covariate_balance() # doctest: +SKIP + """ + if method not in _METHODS: + raise ValueError(f"method must be one of {list(_METHODS)!r}, got {method!r}") + if base_period not in _BASE_PERIODS: + raise ValueError( + f"base_period must be one of {list(_BASE_PERIODS)!r}, got " f"{base_period!r}" + ) + + covariate_names = tuple(covariates or ()) + balance_names = tuple(balance_covariates or ()) + panel = _Panel( + data, + outcome=outcome, + unit=unit, + time=time, + first_treat=first_treat, + covariates=covariate_names, + weights=weights, + ) + balance_block = ( + panel.covariate_block(balance_names, data, unit, time) if balance_names else None + ) + + payload = _decompose_fwl(panel, base_period, balance_names, balance_block) + return TWFEDecompositionResult( + cells=payload["cells"], + method=method, + estimate=payload["estimate"], + decomposition=payload["decomposition"], + remainder=payload["remainder"], + pre_period_contribution=payload["pre_period_contribution"], + post_only=payload["post_only"], + base_period=base_period, + covariates=payload["covariates"], + effective_sample_size=payload["effective_sample_size"], + n_units=panel.n_units, + n_periods=panel.n_periods, + balance=payload["balance"], + ) diff --git a/diff_diff/twfe_weights_results.py b/diff_diff/twfe_weights_results.py new file mode 100644 index 000000000..58a086a65 --- /dev/null +++ b/diff_diff/twfe_weights_results.py @@ -0,0 +1,464 @@ +"""Result containers for the TWFE implicit-weight diagnostics. + +See :mod:`diff_diff.twfe_weights` for the entry points that build these, and +for the upstream MIT attribution. + +Both containers subclass :class:`diff_diff.Diagnostic`: they assess a design +(what a regression implicitly weights) rather than estimating a causal effect, +so neither carries the estimator quintet ``att``/``se``/``t_stat``/``p_value``/ +``conf_int``. The headline scalars are named ``implied_att`` and ``estimate`` +precisely so they do not read as inference-bearing point estimates - the +decomposition is an algebraic identity, exactly like +:class:`~diff_diff.BaconDecompositionResults`. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, Optional, Tuple + +import numpy as np +import pandas as pd + +from diff_diff.results_base import Diagnostic + +__all__ = ["ATTGTWeightsResult", "TWFEDecompositionResult"] + +_AGGREGATION_LABELS = { + "twfe": "TWFE regression", + "overall": "ATT^O (Callaway & Sant'Anna overall)", + "simple": "ATT^simple (Callaway & Sant'Anna simple)", +} + +# Per-cell balance columns, in report order. The three ``_`` -prefixed groups +# mirror R's ``cov_bal_df`` under diff-diff naming; see the mapping table in +# the REGISTRY entry. +_BALANCE_STATS = ( + "unweighted_treated", + "unweighted_control", + "unweighted_diff", + "weighted_treated", + "weighted_control", + "weighted_diff", + "sd", + "unweighted_log_ratio_sd", + "weighted_log_ratio_sd", + "unweighted_frac_extreme", + "weighted_frac_extreme", +) + + +def _fmt(value: float, width: int = 12, digits: int = 4) -> str: + """Right-aligned float that renders NaN without blowing up the layout.""" + if value is None or (isinstance(value, float) and not np.isfinite(value)): + return f"{'n/a':>{width}}" + return f"{value:>{width}.{digits}f}" + + +@dataclass +class ATTGTWeightsResult(Diagnostic): + """Weights that an estimand places on each group-time effect ATT(g, t). + + Returned by :func:`diff_diff.attgt_weights`. One row per ``(g, t)`` cell. + + Attributes + ---------- + weights : pd.DataFrame + Columns ``group``, ``time``, ``post``, ``weight``, ``att``. ``post`` + is ``1`` when ``time >= group`` (the cells the estimand targets), + ``0`` for pre-treatment cells. ``att`` is the ATT(g, t) the weight + multiplies, carried through from the source so that + ``(weight * att).sum()`` reproduces ``implied_att``. + aggregation : str + Which estimand's weights these are: ``"twfe"``, ``"overall"`` + (ATT^O), or ``"simple"`` (ATT^simple). + implied_att : float + ``sum(weight * att)`` - what the estimand delivers given these + ATT(g, t). For ``aggregation="twfe"`` this is the TWFE coefficient. + n_negative : int + Number of cells - PRE and post - receiving a negative weight. Under + ``aggregation="twfe"`` the weights over the full ``g != 0`` grid sum + to zero (post to +1, pre to -1), so this is non-zero in every + staggered design; read ``n_negative_post`` for the pathology. + negative_weight_share : float + ``sum(|w| : w < 0) / sum(|w|)`` over ALL cells - how much of the total + weight mass points the wrong way. Near 0.5 is normal under + ``"twfe"`` for the same reason. ``0.0`` when no weight is negative. + n_negative_post : int + Number of POST-treatment cells receiving a negative weight. This is + the classic staggered-adoption pathology: the regression subtracts + treatment effects it should be adding. Zero for ``"overall"`` and + ``"simple"`` by construction. + negative_post_weight_share : float + ``sum(|w| : w < 0, post) / sum(|w| : post)`` - the share of + post-treatment weight mass that is negative. No R counterpart; see + the methodology registry. + n_cells : int + Number of ``(g, t)`` cells contributing. + source : str or None + ``"CallawaySantAnnaResults"`` when built from a fitted result, + ``"DataFrame"`` on the fallback path. + control_group, base_period : str or None + Design metadata carried from the source fit, when available. + n_dropped_cells : int + Cells excluded because their ATT(g, t) was non-estimable (NaN). + """ + + weights: pd.DataFrame + aggregation: str + implied_att: float + n_negative: int + negative_weight_share: float + n_negative_post: int + negative_post_weight_share: float + n_cells: int + source: Optional[str] = None + control_group: Optional[str] = None + base_period: Optional[str] = None + n_dropped_cells: int = 0 + + def __repr__(self) -> str: + return ( + f"ATTGTWeightsResult(aggregation={self.aggregation!r}, " + f"implied_att={self.implied_att:.4f}, " + f"n_cells={self.n_cells}, n_negative={self.n_negative})" + ) + + def summary(self) -> str: + """Formatted per-cell weight table with the negative-weight roll-up.""" + width = 72 + label = _AGGREGATION_LABELS.get(self.aggregation, self.aggregation) + lines = [ + "=" * width, + "Implicit Weights on ATT(g, t)".center(width), + "=" * width, + "", + f"{'Estimand:':<28} {label}", + f"{'Group-time cells:':<28} {self.n_cells:>10}", + ] + if self.n_dropped_cells: + lines.append(f"{'Non-estimable cells dropped:':<28} {self.n_dropped_cells:>10}") + if self.source is not None: + lines.append(f"{'Source:':<28} {self.source}") + if self.control_group is not None: + lines.append(f"{'Control group:':<28} {self.control_group}") + if self.base_period is not None: + lines.append(f"{'Base period:':<28} {self.base_period}") + lines += [ + "", + "-" * width, + f"{'Group':>8} {'Time':>8} {'Post':>6} {'Weight':>14} {'ATT(g,t)':>14}", + "-" * width, + ] + for row in self.weights.itertuples(index=False): + lines.append( + f"{row.group:>8} {row.time:>8} {int(row.post):>6} " + f"{_fmt(row.weight, 14, 6)} {_fmt(row.att, 14, 6)}" + ) + lines += [ + "-" * width, + "", + f"{'Implied estimate:':<28} {_fmt(self.implied_att)}", + f"{'Negative POST-period cells:':<28} {self.n_negative_post:>12}", + f"{'Negative POST-weight share:':<28} {_fmt(self.negative_post_weight_share)}", + f"{'Negative cells (all):':<28} {self.n_negative:>12}", + f"{'Negative share (all):':<28} {_fmt(self.negative_weight_share)}", + "", + ] + if self.n_negative_post: + lines += [ + "Note: negative POST-period weights mean this estimand subtracts some", + " treatment-period ATT(g, t). Under heterogeneous effects the", + " estimate need not lie in the convex hull of those effects.", + "", + ] + elif self.n_negative: + lines += [ + "Note: the negative weights fall on PRE-treatment cells only, which is", + " how the TWFE weights sum to zero over the full grid; no", + " treatment-period effect is being subtracted.", + "", + ] + lines.append("=" * width) + return "\n".join(lines) + + def print_summary(self) -> None: + """Print :meth:`summary` to stdout.""" + print(self.summary()) + + def to_dataframe(self) -> pd.DataFrame: + """Per-cell weight table (a copy).""" + return self.weights.copy() + + def to_dict(self) -> Dict[str, Any]: + """Serializable view of the result.""" + return { + "aggregation": self.aggregation, + "implied_att": self.implied_att, + "n_cells": self.n_cells, + "n_negative": self.n_negative, + "negative_weight_share": self.negative_weight_share, + "n_negative_post": self.n_negative_post, + "negative_post_weight_share": self.negative_post_weight_share, + "n_dropped_cells": self.n_dropped_cells, + "source": self.source, + "control_group": self.control_group, + "base_period": self.base_period, + "weights": self.weights.to_dict(orient="list"), + } + + +@dataclass +class TWFEDecompositionResult(Diagnostic): + """Decomposition of a TWFE estimate into weighted ATT(g, t). + + Returned by :func:`diff_diff.decompose_twfe_weights`. + + Attributes + ---------- + cells : pd.DataFrame + Columns ``group``, ``time``, ``post``, ``att``, ``weight``, ``ess`` + and ``remainder``. ``weight`` is the implicit weight the regression + places on that cell's ATT(g, t) - R's ``alpha_weight``. + method : str + ``"fwl"`` - Frisch-Waugh-Lovell residual weights from the TWFE + regression. (The only method currently implemented; upstream's AIPW + decomposition is a documented follow-up.) + estimate : float + The estimate being decomposed - ``decomposition + remainder``. + decomposition : float + ``sum(weight * att)`` over all cells, pre and post. + remainder : float + Part of ``estimate`` not attributable to any ATT(g, t) cell. + Identically ``0.0`` except under ``base_period="gmin1"``. + pre_period_contribution : float + ``sum(weight * att)`` over PRE-treatment cells only. This is the sample + contribution of the pre-treatment cells to ``estimate``. It can reflect + differential pre-trends, but it can also be sampling variation, since + the diagnostic carries no inference; read it as diagnostic evidence, + not as proof that the identifying assumption fails in the + post-treatment counterfactual. + post_only : float + ``sum(weight * att)`` over post-treatment cells only. + base_period : str or None + ``"first_period"`` or ``"gmin1"``. + covariates : tuple of str + Covariates the regression adjusted for. Empty tuple when none. + effective_sample_size : float + Weight-concentration roll-up. Small values relative to ``n_units`` + mean the estimate leans on few observations. + n_units, n_periods : int + Panel dimensions. + balance : pd.DataFrame or None + Per-cell implicit covariate balance, populated when + ``balance_covariates=`` was requested. Read it via + :meth:`covariate_balance`. + """ + + cells: pd.DataFrame + method: str + estimate: float + decomposition: float + remainder: float + pre_period_contribution: float + post_only: float + base_period: Optional[str] + covariates: Tuple[str, ...] + effective_sample_size: float + n_units: int + n_periods: int + balance: Optional[pd.DataFrame] = field(default=None) + + def __repr__(self) -> str: + return ( + f"TWFEDecompositionResult(method={self.method!r}, " + f"estimate={self.estimate:.4f}, " + f"pre_period_contribution={self.pre_period_contribution:.4f}, " + f"n_cells={len(self.cells)})" + ) + + def summary(self) -> str: + """Formatted decomposition table with the pre-trend contribution.""" + width = 78 + method_label = { + "fwl": "TWFE regression (Frisch-Waugh-Lovell implicit weights)", + }.get(self.method, self.method) + covs = ", ".join(self.covariates) if self.covariates else "(none)" + lines = [ + "=" * width, + "Decomposition into Group-Time Effects".center(width), + "=" * width, + "", + f"{'Method:':<30} {method_label}", + f"{'Covariates:':<30} {covs}", + ] + if self.base_period is not None: + lines.append(f"{'Base period:':<30} {self.base_period}") + lines += [ + f"{'Units / periods:':<30} {self.n_units} / {self.n_periods}", + f"{'Group-time cells:':<30} {len(self.cells)}", + "", + "-" * width, + f"{'Group':>8} {'Time':>8} {'Post':>6} {'Weight':>14} " + f"{'ATT(g,t)':>14} {'Contribution':>14}", + "-" * width, + ] + for row in self.cells.itertuples(index=False): + lines.append( + f"{row.group:>8} {row.time:>8} {int(row.post):>6} " + f"{_fmt(row.weight, 14, 6)} {_fmt(row.att, 14, 6)} " + f"{_fmt(row.weight * row.att, 14, 6)}" + ) + lines += [ + "-" * width, + "", + f"{'Estimate:':<30} {_fmt(self.estimate)}", + f"{' from ATT(g,t) cells:':<30} {_fmt(self.decomposition)}", + f"{' post-treatment only:':<30} {_fmt(self.post_only)}", + f"{' from pre-period cells:':<30} {_fmt(self.pre_period_contribution)}", + f"{' remainder:':<30} {_fmt(self.remainder)}", + "", + f"{'Effective sample size:':<30} {_fmt(self.effective_sample_size)}", + "", + ] + if abs(self.pre_period_contribution) > 1e-10: + lines += [ + "Note: the pre-period contribution is the sample contribution of", + " the pre-treatment cells. It can reflect differential", + " pre-trends, but it can also be sampling variation - this", + " diagnostic carries no inference - so treat it as evidence", + " about the earlier-period restrictions, not as proof that the", + " identifying assumption fails post-treatment.", + "", + ] + if self.balance is not None: + lines += [ + "Covariate balance available via .covariate_balance().", + "", + ] + lines.append("=" * width) + return "\n".join(lines) + + def print_summary(self) -> None: + """Print :meth:`summary` to stdout.""" + print(self.summary()) + + def to_dataframe(self) -> pd.DataFrame: + """Per-cell decomposition table (a copy).""" + return self.cells.copy() + + def to_dict(self) -> Dict[str, Any]: + """Serializable view of the result.""" + out: Dict[str, Any] = { + "method": self.method, + "estimate": self.estimate, + "decomposition": self.decomposition, + "remainder": self.remainder, + "pre_period_contribution": self.pre_period_contribution, + "post_only": self.post_only, + "base_period": self.base_period, + "covariates": list(self.covariates), + "effective_sample_size": self.effective_sample_size, + "n_units": self.n_units, + "n_periods": self.n_periods, + "cells": self.cells.to_dict(orient="list"), + } + if self.balance is not None: + out["balance"] = self.balance.to_dict(orient="list") + return out + + def covariate_balance( + self, + *, + level: str = "summary", + standardize: bool = True, + post_only: bool = True, + ) -> pd.DataFrame: + """Implicit-weight covariate balance. + + Asks whether the weights the regression implicitly applies actually + balance the covariates across the treated and comparison groups. If + ``weighted_diff`` is no closer to zero than ``unweighted_diff``, the + covariate adjustment is not buying what it appears to. + + Parameters + ---------- + level : {"summary", "cell"}, default "summary" + ``"summary"`` aggregates across ``(g, t)`` cells to one row per + covariate, weighting each cell by its implicit weight. + ``"cell"`` returns the unaggregated per-``(g, t)`` rows. + standardize : bool, default True + Append ``unweighted_std_diff`` / ``weighted_std_diff``, the + differences divided by the pooled standard deviation. These are + a diff-diff addition; R reports the raw differences only. + post_only : bool, default True + Restrict the summary roll-up to post-treatment cells, matching + R's ``mp_covariate_bal_summary_helper``. Ignored when + ``level="cell"``. + + Returns + ------- + pd.DataFrame + One row per covariate (``level="summary"``) or per + ``(group, time, covariate)`` (``level="cell"``). + + Raises + ------ + ValueError + If balance was not requested at compute time, or ``level`` is + not one of the two accepted values. + """ + if self.balance is None: + raise ValueError( + "Covariate balance was not computed for this decomposition. " + "Re-run with balance_covariates=, e.g.\n" + " decompose_twfe_weights(..., balance_covariates=['x1', 'x2'])" + ) + if level not in ("summary", "cell"): + raise ValueError(f"level must be 'summary' or 'cell', got {level!r}") + + table = self.balance.copy() + if level == "cell": + if standardize: + table = _append_standardized(table) + return table + + weights = self.cells.set_index(["group", "time"])["weight"] + keys = pd.MultiIndex.from_arrays([table["group"], table["time"]]) + table["_w"] = weights.reindex(keys).to_numpy() + if post_only: + # Mask on the `post` COLUMN, not on a zero roll-up weight: R's + # post-only helper never touches the pre cells, but a post cell + # whose implicit weight happens to be exactly zero still + # contributes - and still propagates its NA. + table = table[table["post"].to_numpy().astype(bool)] + + # R propagates NA: if ANY contributing cell of a (covariate, statistic) + # is NA, the summary is NA. pandas' sum() skips NaN, which would turn + # the documented NA return of frac_treated_extreme (fewer than three + # distinct values) into a spurious 0.0. + stats = table[list(_BALANCE_STATS)].mul(table["_w"], axis=0) + groups = table["covariate"].to_numpy() + rolled = stats.groupby(groups, sort=False).sum(min_count=1) + any_nan = table[list(_BALANCE_STATS)].isna().groupby(groups, sort=False).any() + rolled = rolled.mask(any_nan) + rolled.index.name = "covariate" + out = rolled.reset_index() + if standardize: + out = _append_standardized(out) + return out + + +def _append_standardized(table: pd.DataFrame) -> pd.DataFrame: + """Add ``*_std_diff`` columns (difference / pooled SD). + + A diff-diff addition on top of R's columns - additive, so parity is + asserted on the R columns only. ``sd == 0`` yields NaN rather than an + infinity, so a degenerate covariate does not poison a summary table. + """ + out = table.copy() + sd = out["sd"].to_numpy(dtype=float) + safe = np.where(sd == 0, np.nan, sd) + out["unweighted_std_diff"] = out["unweighted_diff"].to_numpy(dtype=float) / safe + out["weighted_std_diff"] = out["weighted_diff"].to_numpy(dtype=float) / safe + return out diff --git a/diff_diff/visualization/__init__.py b/diff_diff/visualization/__init__.py index 4c06bb751..8d255f67c 100644 --- a/diff_diff/visualization/__init__.py +++ b/diff_diff/visualization/__init__.py @@ -15,6 +15,7 @@ from diff_diff.visualization._diagnostic import ( plot_bacon, plot_sensitivity, + plot_twfe_weights, ) from diff_diff.visualization._event_study import ( PlottableResults, @@ -48,6 +49,7 @@ "plot_group_effects", "plot_sensitivity", "plot_bacon", + "plot_twfe_weights", "plot_power_curve", "plot_pretrends_power", # New public functions diff --git a/diff_diff/visualization/_diagnostic.py b/diff_diff/visualization/_diagnostic.py index ef2dd05b2..da496f518 100644 --- a/diff_diff/visualization/_diagnostic.py +++ b/diff_diff/visualization/_diagnostic.py @@ -1,12 +1,20 @@ -"""Diagnostic visualization functions (sensitivity, Bacon decomposition).""" +"""Diagnostic visualization functions (sensitivity, Bacon decomposition). -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +``plot_twfe_weights`` renders the implicit ATT(g, t) weights (or their covariate +balance) from :func:`diff_diff.attgt_weights` / :func:`diff_diff.decompose_twfe_weights`. +""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import numpy as np if TYPE_CHECKING: from diff_diff.bacon import BaconDecompositionResults from diff_diff.honest_did import SensitivityResults + from diff_diff.twfe_weights_results import ( + ATTGTWeightsResult, + TWFEDecompositionResult, + ) def plot_sensitivity( @@ -817,3 +825,343 @@ def _render_bacon_plotly( fig.show() return fig + + +def plot_twfe_weights( + results: "Union[ATTGTWeightsResult, TWFEDecompositionResult]", + *, + kind: str = "auto", + standardize: bool = True, + absolute_value: bool = True, + figsize: Tuple[float, float] = (10, 6), + title: Optional[str] = None, + xlabel: Optional[str] = None, + ylabel: Optional[str] = None, + post_color: str = "#2563eb", + pre_color: str = "#dc2626", + markersize: int = 80, + alpha: float = 0.8, + annotate: bool = False, + ax: Optional[Any] = None, + show: bool = True, + backend: str = "matplotlib", +) -> Any: + """Visualize implicit TWFE weights on ATT(g, t), or their covariate balance. + + Two views, matching upstream's ``ggtwfeweights`` methods: + + - ``kind="weights"`` plots weight against ATT(g, t), one point per + group-time cell, coloured by pre/post. Points to the LEFT of the + vertical zero line carry negative weight - the staggered-TWFE + pathology. + - ``kind="balance"`` plots unweighted against implicitly-weighted + covariate differences. Points near zero on the vertical axis are + covariates the implicit weights balance. + + Parameters + ---------- + results : ATTGTWeightsResult or TWFEDecompositionResult + Output of :func:`diff_diff.attgt_weights` or + :func:`diff_diff.decompose_twfe_weights`. + kind : {"auto", "weights", "balance"}, default "auto" + ``"auto"`` picks ``"balance"`` when the result carries a balance + table and ``"weights"`` otherwise. + standardize : bool, default True + Balance view: divide differences by the pooled standard deviation. + absolute_value : bool, default True + Balance view: plot absolute differences, so "closer to zero is + better" reads the same for every covariate. + figsize : tuple, default (10, 6) + Figure size in inches. Ignored when ``ax`` is supplied. + title, xlabel, ylabel : str, optional + Overrides for the defaults chosen per ``kind``. + post_color, pre_color : str + Colors for post- and pre-treatment cells (weights view). + markersize : int, default 80 + Scatter marker area. + alpha : float, default 0.8 + Marker opacity. + annotate : bool, default False + Label each point with its ``(group, time)`` or covariate name. + ax : matplotlib Axes, optional + Axes to draw on. A new figure is created when omitted. Matplotlib only. + show : bool, default True + Call ``plt.show()`` / ``fig.show()`` before returning. + backend : str, default "matplotlib" + Plotting backend: ``"matplotlib"`` or ``"plotly"``. + + Returns + ------- + matplotlib.axes.Axes or plotly.graph_objects.Figure + + Raises + ------ + ValueError + On an unknown ``kind`` or ``backend``; when ``kind="balance"`` is + requested for a result that carries no balance table; or when the + balance table has no finite differences to plot. + + Examples + -------- + >>> import diff_diff # doctest: +SKIP + >>> w = diff_diff.attgt_weights(cs_result) # doctest: +SKIP + >>> diff_diff.plot_twfe_weights(w) # doctest: +SKIP + >>> diff_diff.plot_twfe_weights(w, backend="plotly") # doctest: +SKIP + """ + if kind not in ("auto", "weights", "balance"): + raise ValueError(f"kind must be one of ['auto', 'weights', 'balance'], got {kind!r}") + if backend not in ("matplotlib", "plotly"): + raise ValueError(f"backend must be 'matplotlib' or 'plotly', got {backend!r}") + has_balance = getattr(results, "balance", None) is not None + if kind == "auto": + kind = "balance" if has_balance else "weights" + if kind == "balance" and not has_balance: + raise ValueError( + "this result carries no covariate balance table, so kind='balance' " + "has nothing to plot. Recompute with " + "decompose_twfe_weights(..., balance_covariates=[...])." + ) + + payload = _twfe_weights_payload(results, kind, standardize, absolute_value) + render = _render_twfe_weights_plotly if backend == "plotly" else _render_twfe_weights_mpl + return render( + payload, + kind=kind, + standardize=standardize, + figsize=figsize, + title=title, + xlabel=xlabel, + ylabel=ylabel, + post_color=post_color, + pre_color=pre_color, + markersize=markersize, + alpha=alpha, + annotate=annotate, + ax=ax, + show=show, + ) + + +def _twfe_weights_payload( + results: Any, kind: str, standardize: bool, absolute_value: bool +) -> Dict[str, Any]: + """Backend-agnostic data for either view (shared by the two renderers).""" + if kind == "weights": + table = getattr(results, "weights", None) + if table is None: + table = results.cells + weight = table["weight"].to_numpy(dtype=float) + post = table["post"].to_numpy().astype(bool) + return { + "post": post, + "weight": weight, + "att": table["att"].to_numpy(dtype=float), + "labels": [f"({g}, {t})" for g, t in zip(table["group"], table["time"])], + # POST-only: over the full grid the TWFE weights sum to zero, so a + # healthy staggered panel would still count ~half its cells as + # negative. The pathology is negative weight on post cells. + "n_negative_post": int(((weight < 0) & post).sum()), + } + balance = results.covariate_balance(level="summary", standardize=standardize) + suffix = "_std_diff" if standardize else "_diff" + unweighted = balance["unweighted" + suffix].to_numpy(dtype=float) + weighted = balance["weighted" + suffix].to_numpy(dtype=float) + if absolute_value: + unweighted, weighted = np.abs(unweighted), np.abs(weighted) + finite = np.isfinite(unweighted) & np.isfinite(weighted) + if not finite.any(): + raise ValueError( + "the covariate balance table has no finite differences to plot " + "(every covariate is degenerate - e.g. constant or binary with a zero " + "pooled SD under standardize=True); try standardize=False or check the " + "balance_covariates" + ) + limit = float(np.max(np.abs(np.concatenate([unweighted[finite], weighted[finite]]))) or 1.0) + return { + "unweighted": unweighted, + "weighted": weighted, + "labels": [str(c) for c in balance["covariate"]], + "limit": limit, + "absolute_value": absolute_value, + } + + +def _render_twfe_weights_mpl( + payload: Dict[str, Any], + *, + kind: str, + standardize: bool, + figsize: Tuple[float, float], + title: Optional[str], + xlabel: Optional[str], + ylabel: Optional[str], + post_color: str, + pre_color: str, + markersize: int, + alpha: float, + annotate: bool, + ax: Optional[Any], + show: bool, +) -> Any: + from diff_diff.visualization._common import _require_matplotlib + + plt = _require_matplotlib() + if ax is None: + _, ax = plt.subplots(figsize=figsize) + + if kind == "weights": + post, weight, att = payload["post"], payload["weight"], payload["att"] + ax.axhline(0, color="0.4", linewidth=1.2, zorder=1) + ax.axvline(0, color="0.4", linewidth=1.2, zorder=1) + for mask, color, label in ( + (post, post_color, "post-treatment"), + (~post, pre_color, "pre-treatment"), + ): + if mask.any(): + ax.scatter( + weight[mask], + att[mask], + s=markersize, + alpha=alpha, + color=color, + label=label, + zorder=3, + ) + if annotate: + for w, a, lab in zip(weight, att, payload["labels"]): + ax.annotate(lab, (w, a), fontsize=8, xytext=(4, 4), textcoords="offset points") + ax.set_xlabel(xlabel or "Implicit weight") + ax.set_ylabel(ylabel or "ATT(g, t)") + default_title = "Implicit weights on group-time effects" + if payload["n_negative_post"]: + default_title += f" ({payload['n_negative_post']} negative post)" + ax.set_title(title or default_title) + ax.legend(frameon=False) + else: + unweighted, weighted, limit = payload["unweighted"], payload["weighted"], payload["limit"] + ax.axhline(0, color="0.4", linewidth=1.2, zorder=1) + ax.scatter(unweighted, weighted, s=markersize, alpha=alpha, color=post_color, zorder=3) + # The "no improvement" diagonal: on a SIGNED plot differences can be + # negative, so the line must span [-limit, limit], not [0, limit]. + diagonal_lo = 0.0 if payload["absolute_value"] else -limit + ax.plot( + [diagonal_lo, limit], + [diagonal_lo, limit], + color="0.6", + linestyle="--", + linewidth=1.0, + zorder=2, + label="no improvement", + ) + if annotate: + for x, y, name in zip(unweighted, weighted, payload["labels"]): + ax.annotate(name, (x, y), fontsize=8, xytext=(4, 4), textcoords="offset points") + kindword = "standardized " if standardize else "" + ax.set_xlabel(xlabel or f"Unweighted {kindword}difference") + ax.set_ylabel(ylabel or f"Implicitly-weighted {kindword}difference") + ax.set_title(title or "Covariate balance under the implicit weights") + ax.legend(frameon=False) + + if show: + plt.show() + return ax + + +def _render_twfe_weights_plotly( + payload: Dict[str, Any], + *, + kind: str, + standardize: bool, + figsize: Tuple[float, float], + title: Optional[str], + xlabel: Optional[str], + ylabel: Optional[str], + post_color: str, + pre_color: str, + markersize: int, + alpha: float, + annotate: bool, + ax: Optional[Any], + show: bool, +) -> Any: + from diff_diff.visualization._common import _require_plotly + + go = _require_plotly() + fig = go.Figure() + marker_px = max(4.0, float(np.sqrt(markersize))) # matplotlib area -> plotly diameter + mode = "markers+text" if annotate else "markers" + + if kind == "weights": + post, weight, att = payload["post"], payload["weight"], payload["att"] + labels = np.asarray(payload["labels"], dtype=object) + for mask, color, name in ( + (post, post_color, "post-treatment"), + (~post, pre_color, "pre-treatment"), + ): + if mask.any(): + fig.add_trace( + go.Scatter( + x=weight[mask], + y=att[mask], + mode=mode, + name=name, + # `text` always carries the labels: with mode="markers" + # they feed the hover template; with "markers+text" + # (annotate=True) they also render. + text=labels[mask], + textposition="top right", + marker={"size": marker_px, "color": color, "opacity": alpha}, + hovertemplate="%{text}
weight=%{x:.4f}
ATT(g,t)=%{y:.4f}", + ) + ) + fig.add_hline(y=0, line={"color": "gray", "width": 1.2}) + fig.add_vline(x=0, line={"color": "gray", "width": 1.2}) + default_title = "Implicit weights on group-time effects" + if payload["n_negative_post"]: + default_title += f" ({payload['n_negative_post']} negative post)" + fig.update_layout( + title=title or default_title, + xaxis_title=xlabel or "Implicit weight", + yaxis_title=ylabel or "ATT(g, t)", + ) + else: + unweighted, weighted, limit = payload["unweighted"], payload["weighted"], payload["limit"] + fig.add_trace( + go.Scatter( + x=unweighted, + y=weighted, + mode=mode, + name="covariates", + text=payload["labels"], + textposition="top right", + marker={"size": marker_px, "color": post_color, "opacity": alpha}, + hovertemplate="%{text}
unweighted=%{x:.4f}
weighted=%{y:.4f}", + ) + ) + diagonal_lo = 0.0 if payload["absolute_value"] else -limit + fig.add_trace( + go.Scatter( + x=[diagonal_lo, limit], + y=[diagonal_lo, limit], + mode="lines", + name="no improvement", + line={"color": "gray", "dash": "dash", "width": 1.0}, + ) + ) + fig.add_hline(y=0, line={"color": "gray", "width": 1.2}) + kindword = "standardized " if standardize else "" + fig.update_layout( + title=title or "Covariate balance under the implicit weights", + xaxis_title=xlabel or f"Unweighted {kindword}difference", + yaxis_title=ylabel or f"Implicitly-weighted {kindword}difference", + ) + + fig.update_layout( + width=int(figsize[0] * 100), + height=int(figsize[1] * 100), + template="plotly_white", + legend={"orientation": "h", "y": -0.15}, + ) + if show: + fig.show() + return fig diff --git a/docs/api/index.rst b/docs/api/index.rst index ad05dee06..5c02bb7ec 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -77,6 +77,8 @@ Result containers returned by estimators: diff_diff.TwoStageBootstrapResults diff_diff.SpilloverDiDResults diff_diff.BaconDecompositionResults + diff_diff.ATTGTWeightsResult + diff_diff.TWFEDecompositionResult diff_diff.wooldridge_results.WooldridgeDiDResults diff_diff.lpdid_results.LPDiDResults diff_diff.changes_in_changes_results.ChangesInChangesResults @@ -122,6 +124,7 @@ Plotting functions and plot builders: diff_diff.plot_honest_event_study diff_diff.RDPlot diff_diff.plot_bacon + diff_diff.plot_twfe_weights diff_diff.plot_power_curve diff_diff.plot_pretrends_power @@ -141,6 +144,8 @@ Placebo tests and model diagnostics: diff_diff.leave_one_out_test diff_diff.run_all_placebo_tests diff_diff.PlaceboTestResults + diff_diff.attgt_weights + diff_diff.decompose_twfe_weights diff_diff.RDDensityTest Panel Profiling @@ -404,6 +409,7 @@ Diagnostics & Inference honest_did power pretrends + twfe_weights Reporting ~~~~~~~~~ diff --git a/docs/api/twfe_weights.rst b/docs/api/twfe_weights.rst new file mode 100644 index 000000000..9b39b31f0 --- /dev/null +++ b/docs/api/twfe_weights.rst @@ -0,0 +1,161 @@ +TWFE Weight Diagnostics (Callaway ``twfeweights``) +=================================================== + +What a two-way fixed effects regression *implicitly* weights. + +Run on staggered-adoption data, a TWFE regression does not estimate a simple +average of the underlying group-time effects ATT(g, t). It estimates a +weighted average, and some of those weights can be **negative** -- so the +coefficient need not lie in the convex hull of the effects it summarizes. +This module reports those weights, next to the weights the target estimands +ATT\ :sup:`O` and ATT\ :sup:`simple` would use, and decomposes the regression +back into its building blocks. + +**When to use these diagnostics:** + +- You have a staggered design and want to see, cell by cell, what your TWFE + specification is actually averaging +- You want to quantify what the *pre-treatment* cells contribute to a TWFE + estimate -- a sample contribution that can reflect parallel-trends + violations, but can also be sampling variation, so read it as diagnostic + evidence rather than proof +- Your TWFE and :class:`~diff_diff.CallawaySantAnna` estimates disagree and + you want to see which cells drive the gap +- You adjusted for covariates and want to check whether the regression's + implicit weights actually *balance* them + +**How this differs from the neighbouring surfaces:** + +- :func:`diff_diff.twowayfeweights` implements de Chaisemartin & + D'Haultfoeuille (2020) Theorem 1 and weights **(unit, time) cells**. The + functions here weight **ATT(g, t) parameters**. +- :class:`diff_diff.BaconDecomposition` decomposes TWFE into **2x2 DiD + comparisons**. :func:`diff_diff.decompose_twfe_weights` decomposes it into + **group-time effects**, plus a pre-trend-violation term. + +**Reference:** Baker, A., Callaway, B., Cunningham, S., Goodman-Bacon, A., & +Sant'Anna, P. H. C. (2025). Difference-in-Differences Designs: A +Practitioner's Guide. arXiv:2503.13323. Callaway, B., & Sant'Anna, P. H. C. +(2021) for the ATT\ :sup:`O` / ATT\ :sup:`simple` weights. + +Ported from the ``twfeweights`` R package (v0.9.0) by Brantly Callaway, MIT +License, Copyright (c) 2023 Brantly Callaway. + +.. module:: diff_diff.twfe_weights + +attgt_weights +------------- + +Weights an estimand places on each group-time effect. + +.. autofunction:: diff_diff.attgt_weights + +decompose_twfe_weights +---------------------- + +Decomposition of a TWFE estimate into weighted group-time effects. + +.. autofunction:: diff_diff.decompose_twfe_weights + +plot_twfe_weights +----------------- + +.. autofunction:: diff_diff.plot_twfe_weights + +Result Objects +-------------- + +.. autoclass:: diff_diff.ATTGTWeightsResult + :members: + :undoc-members: + :show-inheritance: + :no-index: + +.. autoclass:: diff_diff.TWFEDecompositionResult + :members: + :undoc-members: + :show-inheritance: + :no-index: + +Example Usage +------------- + +Inspecting what a TWFE regression weights +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import diff_diff + + panel = diff_diff.load_mpdta() + + cs = diff_diff.CallawaySantAnna( + control_group="never_treated", + base_period="universal", # required for aggregation="twfe" + ).fit( + panel, outcome="lemp", unit="countyreal", time="year", + first_treat="first_treat", + ) + + weights = diff_diff.attgt_weights(cs, aggregation="twfe") + print(weights.summary()) + print(weights.n_negative, "cells carry negative weight") + +Comparing against the estimand you meant to report +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``aggregation="overall"`` and ``"simple"`` give the Callaway & Sant'Anna +target-parameter weights, which are non-negative and sum to one. The gap +between ``implied_att`` values is the cost of the TWFE specification: + +.. code-block:: python + + import diff_diff + + panel = diff_diff.load_mpdta() + cs = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal", + ).fit( + panel, outcome="lemp", unit="countyreal", time="year", + first_treat="first_treat", + ) + + for aggregation in ("twfe", "overall", "simple"): + w = diff_diff.attgt_weights(cs, aggregation=aggregation) + print(f"{aggregation:8s} {w.implied_att: .4f} " + f"({w.n_negative} negative weights)") + +Separating treatment effects from the pre-period contribution +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import diff_diff + + panel = diff_diff.load_mpdta() + + decomposition = diff_diff.decompose_twfe_weights( + panel, + outcome="lemp", unit="countyreal", time="year", + first_treat="first_treat", + covariates=["lpop"], + balance_covariates=["lpop"], + ) + + print(decomposition.summary()) + print("from pre-treatment cells:", decomposition.pre_period_contribution) + + # Do the implicit weights balance the covariates? + print(decomposition.covariate_balance()) + + diff_diff.plot_twfe_weights(decomposition) + +Validation +---------- + +Validated against R ``twfeweights`` 0.9.0 output on three fixtures (``mpdta`` +plus two simulated panels). Goldens live at +``benchmarks/data/twfeweights_golden.json`` and are regenerated with +``Rscript benchmarks/R/generate_twfeweights_golden.R``; R is never needed to +run the test suite. Tolerances and their rationale are in +``docs/methodology/REGISTRY.md`` under "TWFE Weight Diagnostics". diff --git a/docs/api/visualization.rst b/docs/api/visualization.rst index b143d2d20..2b03605a0 100644 --- a/docs/api/visualization.rst +++ b/docs/api/visualization.rst @@ -194,6 +194,12 @@ plot_bacon Visualize Goodman-Bacon decomposition results. +.. seealso:: + + :func:`diff_diff.plot_twfe_weights` renders the implicit ATT(g, t) weights + and their covariate balance. It is documented on its own page, + :doc:`twfe_weights`, and supports the same ``backend=`` options. + .. autofunction:: diff_diff.plot_bacon Example diff --git a/docs/doc-deps.yaml b/docs/doc-deps.yaml index eac062e9c..a5046c8f3 100644 --- a/docs/doc-deps.yaml +++ b/docs/doc-deps.yaml @@ -21,6 +21,9 @@ # Members resolve to the first entry (the primary module) for doc lookup. # ────────────���───────────────────────────────────────────────────────── groups: + twfe_weights: + - diff_diff/twfe_weights.py + - diff_diff/twfe_weights_results.py staggered: - diff_diff/staggered.py - diff_diff/staggered_aggregation.py @@ -1201,6 +1204,36 @@ sources: - path: docs/r_comparison.rst type: user_guide + # ── TWFE Weight Diagnostics ───���───────────────────────────────────────── + + diff_diff/twfe_weights.py: + drift_risk: low + docs: + - path: docs/methodology/REGISTRY.md + section: "TWFE Weight Diagnostics" + type: methodology + - path: docs/api/twfe_weights.rst + type: api_reference + - path: README.md + section: "Diagnostics & Sensitivity (one-line catalog entry)" + type: user_guide + - path: docs/references.rst + type: user_guide + - path: diff_diff/guides/llms.txt + section: "Diagnostics and Sensitivity Analysis" + type: user_guide + - path: diff_diff/guides/llms-full.txt + section: "TWFE Weight Diagnostics" + type: user_guide + diff_diff/twfe_weights_results.py: + drift_risk: low + docs: + - path: docs/api/twfe_weights.rst + type: api_reference + - path: docs/methodology/REGISTRY.md + section: "TWFE Weight Diagnostics" + type: methodology + # ── BaconDecomposition ───���───────────────────────────────────────── diff_diff/bacon.py: diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 7d4886fff..60478b08f 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -39,6 +39,7 @@ This document provides the academic foundations and key implementation requireme 5. [Diagnostics and Sensitivity](#diagnostics-and-sensitivity) - [PlaceboTests](#placebotests) - [BaconDecomposition](#bacondecomposition) + - [TWFE Weight Diagnostics](#twfe-weight-diagnostics) - [HonestDiD](#honestdid) - [PreTrendsPower](#pretrendspower) - [PowerAnalysis](#poweranalysis) @@ -6211,6 +6212,136 @@ Where `n_k` is the sample share of timing group `k`, `n_{kℓ} = n_k / (n_k + n_ --- +## TWFE Weight Diagnostics + +**Primary source:** [Baker, A., Callaway, B., Cunningham, S., Goodman-Bacon, A., & Sant'Anna, P. H. C. (2025). "Difference-in-Differences Designs: A Practitioner's Guide." arXiv:2503.13323](https://arxiv.org/abs/2503.13323) + +**Secondary source:** [Callaway, B., & Sant'Anna, P. H. C. (2021). Difference-in-Differences with multiple time periods. *Journal of Econometrics*, 225(2), 200-230.](https://doi.org/10.1016/j.jeconom.2020.12.001) — for the ATT^O / ATT^simple target-parameter weights. + +**Reference implementation:** the `twfeweights` R package (v0.9.0) by Brantly Callaway, MIT License, Copyright (c) 2023 Brantly Callaway. The upstream notice is reproduced in the module docstring of `diff_diff/twfe_weights.py`, as its terms require. + +**Scope:** these are DIAGNOSTICS, not estimators. Both result containers subclass `Diagnostic` and carry no inference quintet — the decomposition is an algebraic identity, so there is nothing to attach a standard error to. The headline scalars are named `implied_att` and `estimate` rather than `att` for the same reason. + +### Relationship to neighbouring surfaces + +- **vs `twowayfeweights` (de Chaisemartin & D'Haultfoeuille 2020, Theorem 1):** that surface weights **(unit, time) cells**; these functions weight **ATT(g,t) parameters** — the cohort-by-period building blocks. Both detect negative weighting in staggered TWFE, but they decompose along different axes and their weight tables are not comparable row-for-row. The names are deliberately disjoint (`attgt_weights` / `ATTGTWeightsResult` vs `twowayfeweights` / `TWFEWeightsResult`). +- **vs `BaconDecomposition` (Goodman-Bacon 2021):** Bacon decomposes TWFE into **2x2 DiD comparisons** and asks which comparisons drive the estimate. `decompose_twfe_weights` decomposes it into **group-time effects** and additionally isolates a pre-trend-violation term. Use Bacon to see the forbidden comparisons; use this to see the per-`(g,t)` weights and how much of the estimate is not a treatment effect at all. + +### Estimator equations (as implemented) + +All expressions are evaluated in POSITIONAL time (periods mapped to `1..T`, cohorts to their period position, never-treated staying `0`), so `maxT == T`. + +*ATT(g,t) weights — `attgt_weights(aggregation=...)`:* + +`aggregation="twfe"` (R `twfe_weights`), with `p_g` the share of ALL units in cohort `g` and `E_t[D]` the share of units treated by `t`: + +``` +h(g,t) = 1[t >= g] - (maxT - g + 1)/T - E_t[D] + mean_t E_t[D] +num(g,t) = h(g,t) * p_g +w(g,t) = num(g,t) / sum over {t >= g, g != 0} of num(g,t) +``` + +`aggregation="overall"` (ATT^O, R `attO_weights`), with `pbar_g` the share of EVER-TREATED units in cohort `g`: + +``` +w(g,t) = 1[t >= g] * pbar_g / (maxT - g + 1) +``` + +`aggregation="simple"` (ATT^simple, R `att_simple_weights`): + +``` +w(g,t) = 1[t >= g] * pbar_g, then normalized to sum to one +``` + +*FWL decomposition — `decompose_twfe_weights(method="fwl")` (R `implicit_twfe_weights`):* + +Double-demean the treatment indicator `D` and the covariates `X` over unit and period, project the demeaned treatment on the demeaned covariates, and take the residual: + +``` +gamma = argmin_b || Ddot - Xdot b ||_w +resid = Ddot - Xdot gamma +alpha_den = E_w[resid * Ddot] +``` + +The residual IS the implicit weight the regression applies to each observation. Per `(g,t)` cell, with the treated and comparison weights each normalized to mean one: + +``` +alpha_weight(g,t) = E_w[resid | G=g, T=t] * p_g / (alpha_den * T) +ATT(g,t) = E_w[wtreated * Ytilde | G=g] - E_w[wcontrol * Ytilde | G=0] +``` + +where `Ytilde` is the outcome measured against the base period (`Y_t - Y_1` under `base_period="first_period"`, `Y_t - Y_{g-1}` under `"gmin1"`). Roll-ups: + +``` +decomposition = sum over all cells of alpha_weight * ATT +remainder = sum of alpha_weight * cell remainder (0 unless base_period="gmin1") +estimate = decomposition + remainder +pre_period_contribution = sum over PRE cells (t < g) of alpha_weight * ATT +``` + +`pre_period_contribution` is the sample contribution of the pre-treatment cells to `estimate`. It can reflect differential pre-trends, but it can also be sampling variation - the diagnostic carries no inference - so it is evidence about the earlier-period restrictions, not proof that the identifying assumption fails in the post-treatment counterfactual. + +*Cross-surface identity (pinned by `tests/test_twfe_weights_parity.py::TestCrossSurfaceIdentity`):* when the CS fit used `base_period="universal"`, `control_group="never_treated"` and no covariates, + +``` +attgt_weights(cs, aggregation="twfe").implied_att == decompose_twfe_weights(panel, ...).estimate +``` + +Verified on `mpdta` at `-0.03654894` from both directions. + +### Edge cases + +- **Note (grid completeness is a hard error):** `attgt_weights` fails closed on an incomplete group-time grid: a weight table over a partial grid is not the named estimand. `aggregation="twfe"` requires every cohort x period cell (pre cells enter `h(g,t)`); `"overall"` / `"simple"` require every post-treatment cell. The error names the missing cells and each cell's CS `skip_reason`. Only the two structural gaps below are exempt. A non-estimable PRE cell of a surviving cohort under `"overall"` / `"simple"` is still merely dropped-and-warned and counted in `n_dropped_cells`: those estimands place no weight on pre cells, so nothing renormalizes and no number moves. +- **Note (matches R `did`'s first-period drop, but only when the drop is structural):** a cohort with NO estimable post-treatment cell is excluded from the weight table AND from the cohort masses (`p_g`, `pbar_g`, `E_t[D]`) with a `UserWarning`, exactly as `did::pre_process_did` drops units already treated in the first period. The criterion is *post* cells, not all cells: a cohort can have an estimable universal-base pre cell and still no usable post cell. The drop is allowed ONLY when the cohort is treated in the first observed period, OR when every one of its missing post cells carries `skip_reason="zero_treated_control"` (it ran out of comparison units). A mid cohort blanked out any other way (e.g. NaN effects with `skip_reason=None`) raises instead of disappearing. On a bare ATT(g,t) frame with no `skip_reason` column, only a cohort whose label equals the first observed period is excludable this way; any other absent cohort raises, being indistinguishable from user truncation. +- **Note (matches R `did`'s mechanism behind the second structural gap):** with no never-treated group on a `control_group="not_yet_treated"` fit, `did::pre_process_did` truncates the panel to periods `t < max(g)` and recodes the last-treated cohort to never-treated. That truncation - not `aggte` averaging over available periods - is what produces R's number. diff-diff reaches the same numerics by treating the cells CS marks `skip_reason="zero_treated_control"` as structurally ABSENT (they name exactly the cohort x period cells that truncation removes): `"overall"` divides each cohort by its number of available post periods instead of `(maxT - g + 1)`, and `"simple"` renormalizes over the available post cells. A `UserWarning` names the cells. The carve-out is keyed on the `skip_reason` VALUE rather than on `control_group`, because that reason is only ever emitted on a not-yet-treated fit and the frame path carries no `control_group`; this is what makes the fitted and frame paths behave identically. `"twfe"` requires a never-treated comparison group and never reaches this branch; a bare frame without `skip_reason` stays strict. +- **Note (cohort labels are validated):** never-treated is exactly `0` or `+inf`. Any other non-finite label (NaN, `-inf`) raises rather than being silently absorbed into cohort 0; before this check a single NaN label moved `decompose_twfe_weights(...).estimate` by ~1.4% with no warning. Within-unit invariance uses `nunique(dropna=False)` so a label that is NaN in one period fails, and non-finite period labels are rejected up front. +- **Note (`aggregation="twfe"` requires an unadjusted fit):** R's `twfe_weights` stops unless `xformla == ~1`. The fit records its covariate column names on the aggregation kit (`bookkeeping["covariates"]`) and a non-empty tuple raises. A kit predating that bookkeeping warns instead; a bare ATT(g,t) frame carries no record at all, so on the DataFrame path the caller is responsible (stated in the docstring). Use `decompose_twfe_weights(covariates=...)` for the covariate-adjusted decomposition. +- **Note (sampling weights are validated):** unit weights must be finite, non-negative, with positive total and positive treated mass. Positive never-treated mass is required only where the comparison group enters the formula, i.e. `aggregation="twfe"` and `decompose_twfe_weights`, never for ATT^O / ATT^simple, which are defined without a never-treated group. +- `decompose_twfe_weights` requires a balanced panel and a never-treated comparison group, and rejects time-varying cohort labels or sampling weights. +- `base_period="gmin1"` requires a period before each cohort's treatment; a cohort treated in the first period raises. +- `attgt_weights` rejects repeated-cross-section fits, unbalanced-panel fallbacks, and any fitted result whose kit records `is_balanced = False`: `E_t[D]` and the cohort shares average over a fixed unit set. The frame path requires exactly one observation per unit-period (enforced by `_unit_cohorts_from_frame`'s balance check). +- **Note (the anticipation window is honoured on the CS estimands):** the fitted path reads `anticipation` off the aggregation kit; the frame path takes it as an explicit `anticipation=` argument (a bare frame carries no record of the source fit's setting, and the fitted path rejects the kwarg rather than silently ignoring it). Post cells become `t >= g - anticipation` for `"overall"` / `"simple"`, and the same window enters the required-cell grid, the available-period divisors, and the `post` column. `"twfe"` deliberately keeps `t >= g`: the regression's own treatment indicator does not anticipate, and R's `twfe_weights` has no anticipation argument. Each CS estimand is pinned against `results.aggregate(...)` with positive anticipation under both control groups (`"simple"` against `aggregate("simple")`, `"overall"` against the cohort-mass-weighted combination of `aggregate("group")`). +- **Note (one canonical numeric time key):** period labels are coerced to a single numeric key at validation, and that key is used for sorting, reshaping, cohort mapping, and the positional grid. Using the raw column instead lets a numeric-string label (`"10"` sorts before `"2"` lexicographically) desynchronize the sort order from the grid and silently rebuild a different panel; numeric and numeric-string labels now decompose identically. +- **Note (inputs are validated finite):** `decompose_twfe_weights` fails closed on non-finite outcomes, regression covariates, or balance covariates. A single NaN outcome would otherwise propagate to every cell and return an all-NaN decomposition with no error; complete-case handling is a policy choice the caller owns. +- **Note (a wrong-length `weights=` fails closed):** the length check runs before the excluded-cohort boolean slice, so a wrong-length vector is a clear `ValueError` naming both lengths rather than a raw `IndexError`. + +### Notes and deviations + +- **Note (upstream `fixest::demean` segfault on the no-covariate branch):** `twfeweights::implicit_twfe_weights(xformula = ~1)` builds `model.matrix(~-1, data)`, an `nT x 0` matrix, and `fixest::demean()` SEGFAULTS on a zero-column matrix (reproduced in isolation on R 4.6.1 / fixest 0.14.2: `fixest::demean(matrix(numeric(0), 10, 0), ids)` → `*** caught segfault *** memory not mapped`). This is a zero-column bug, not a property of any fixture. The no-covariate golden is therefore generated with a TIME-INVARIANT covariate, which double-demeaning annihilates exactly, making the call numerically the `~1` branch; the parity test asserts BOTH `covariates=None` and `covariates=[]` against that single golden, so the equivalence is proven rather than assumed. Verified on `mpdta`: `twfe_weights(att_gt(...))` aggregate and `implicit_twfe_weights(xformula = ~lpop)$est` both equal `-0.03654894`. +- **Note (annihilated covariates are dropped before the projection, as numerical hygiene):** a covariate with no within-unit-and-period variation leaves a column of pure rounding noise after double-demeaning (~1e-16 against a raw scale of ~1). Keeping it is not catastrophic: the column lies in the fixed-effect span and is orthogonal to the treatment residual, so on mpdta's `lpop` it moves the FWL residual by ~2e-18. But regressing on an exactly-zero column is meaningless, and dropping it is what makes `covariates=None` and `covariates=[]` agree exactly rather than approximately. diff-diff judges each column against its own PRE-demeaning norm; a rank test on the demeaned matrix alone cannot see this, because there 1e-16 is simply the largest pivot. The threshold is the accumulated ROUNDING-NOISE scale - `sqrt(n_obs) * 64 * eps * max(raw_scale, 1)` - not a fixed relative constant: a fixed `1e-10` is five orders too loose (a covariate with level `1e6` and genuine within-sd `1e-4` has ratio `1e-10` and would be wrongly discarded) while still annihilating mpdta's `lpop`, whose true within-variation is far above the noise floor. When a column IS dropped the result reports the SURVIVING covariate names, not the user's input list, so `result.covariates` never claims a column that was annihilated. The `UserWarning` names the dropped columns. +- **Deviation from R (0/0 cells report the limit, not the rounding noise):** for the never-treated comparison group the double-demeaned treatment is CONSTANT within a period (`-E_t[D] + mean_t E_t[D]`), and for some cohort structures that constant is analytically ZERO — on the `sim_staggered` fixture (three equal cohorts at `g in {0,3,4}`, `T=5`) it vanishes exactly at `t=3`, where `-1/3 + 1/3 = 0`. The cell's implicit weights are then `0/0`. diff-diff returns the limit (a constant divided by its own mean is one), giving the plain unweighted contrast; R divides the two rounding errors and lands ~3e-4 away. Verified against a hand-computed contrast that uses none of this module's machinery: diff-diff is exact to 4.4e-16. A `UserWarning` names the affected cells. **`estimate` is unaffected either way** — the weights on such cells cancel exactly (on `sim_staggered`, `w(3,3) + w(4,3) = 0`), so it matches R to 1e-15. Other user-visible fields DO move, because the cancelling cells straddle the pre/post split. Measured against the pinned R values on `sim_staggered`: `pre_period_contribution` and `post_only` each by ~1.2e-4 (equal and opposite, so their sum stays exact), `effective_sample_size` by ~0.99, per-cell `ess` by up to ~0.53, and per-cell `remainder` (under `base_period="gmin1"`) correspondingly. All print in `summary()`. The parity suite asserts every one: tight wherever the degeneracy is not detected, and at the degenerate cells against R's own weights with our limit value substituted only where R's number is 0/0 noise. +- **Deviation from R (positional time rescaling in `attgt_weights`):** R evaluates `(maxT - g + 1) / length(tlist)` on the RAW period labels, which is only correct when those labels are consecutive integers. diff-diff maps periods to `1..T` first (mirroring `BMisc::orig2t`, which R already applies inside `implicit_twfe_weights` but not inside `twfe_weights`). Bit-identical on consecutive grids — `mpdta`'s 2003..2007 maps to 1..5 and both give `4/5` at `g = 2004` — and correct on gapped ones. Pinned by a test that remaps periods to 10, 20, 30, 40, 50. +- **Deviation from R (`keep_untreated` not exposed):** R's `keep_untreated=TRUE` synthesizes `G = 0` rows with `attgt = 0` to mirror an internal vector layout. Those rows are excluded from every normalization (`cond <- .t >= .group & .group != 0`) and contribute exactly zero, so the argument is numerically inert. +- **Deviation from R (consolidated API):** upstream exports 21 symbols in a flat namespace. diff-diff exposes five: `attgt_weights` (folding `twfe_weights` / `attO_weights` / `att_simple_weights` behind `aggregation=`), `decompose_twfe_weights` (folding `implicit_twfe_weights` behind `method=`), the two result classes, and `plot_twfe_weights` (replacing `ggtwfeweights`). The per-cell helpers and the eleven balance statistics are private and pinned through the public surfaces that expose them. The two two-period kernels (`two_period_reg_weights` / `two_period_aipw_weights`) and the AIPW blocks are captured in the golden but read by no test: they are labelled **reserved** in the JSON `meta` and the generator header, pinned so the `method="aipw"` follow-up needs no R re-run. The AIPW golden is covariate-adjusted, since a time-invariant covariate is annihilated by double-demeaning but is NOT a no-op in a propensity score. +- **Deviation from R (post-lasso block out of scope):** `did_post_lasso` / `did_post_lasso_ra` are not ported. The upstream source is unfinished — `R/did_post_lasso.R:69` contains a leftover `browser()` call and references undefined variables — so there is no runnable reference to validate against, and it would add an sklearn dependency. +- **Deviation from R (`method="aipw"` not yet implemented):** upstream's `implicit_aipw_weights` is out of scope for the initial port; `method=` currently accepts `"fwl"` only and raises listing the accepted values. +- **Note (`log_ratio_sd` scaling preserved verbatim):** upstream scales each group's standard deviation by `sqrt(n - 1)` before taking the log ratio, which is not a conventional standard deviation. Preserved as-is for parity; the quantity is only read as a relative balance statistic and the factor largely cancels in the ratio. +- **Note (`frac_treated_extreme` is a step function):** upstream routes through `BMisc::weighted_ecdf` → `make_dist` (an `approxfun(method="constant")` classed as `ecdf`) → `stats:::quantile.ecdf`, which does NOT invert the step function but rebuilds a pseudo-sample by repeating each knot `diff(c(0, round(nobs * F)))` times and takes an ordinary type-7 quantile of that. diff-diff reproduces this exactly, including the `NA` return when the covariate has fewer than three distinct values, and that `NA` survives the summary roll-up. R propagates NA if any contributing cell is NA, whereas a plain pandas `.sum()` skips it and would report a spurious `0.0` for a binary or constant covariate; the roll-up therefore masks on the `post` column (not on a zero roll-up weight, since a post cell whose implicit weight is exactly zero still contributes) and returns NaN whenever any contributing post cell is NaN. Because the statistic is a step function of a weighted ECDF, a perturbation of order 1e-12 can move one unit across a knot and shift the value by `1/n`; parity is gated accordingly. +- **Note (negative-weight statistics; no R counterpart):** over the `g != 0` grid the TWFE weights sum to zero, post cells to +1 and pre cells to -1, so `n_negative` is non-zero and `negative_weight_share` sits near 0.5 in EVERY staggered design, including one with no negative post-period weight. The pathology the literature describes is negative weight on POST cells, so `ATTGTWeightsResult` reports both, labelled: `n_negative` / `negative_weight_share` over all cells, and `n_negative_post` / `negative_post_weight_share` restricted to post cells (the share of post-period weight MASS that is negative). `summary()` leads with the post-only figures. R reports neither statistic. +- **Note (weighted `aggregation="twfe"` is a diff-diff extension):** R's `twfe_weights` takes no `w=`, so there is no upstream reference for a weighted TWFE weight table. The equations above define `p_g` and `E_t[D]` as unweighted shares; with `weights=` they become the corresponding weighted shares (each unit's mass is its sampling weight rather than one), the same algebra a weighted TWFE regression implies on a balanced panel. Pinned by asserting that `attgt_weights(fit, aggregation="twfe", weights=w)` and `decompose_twfe_weights(panel, weights="w")` produce identical weight vectors to 1e-12, plus a frozen-numbers regression test on a synthetic weighted panel (every parity fixture is unweighted, so R cannot gate this path). +- **Note (linear algebra runs through the house helpers):** the two-way demeaning is `diff_diff.utils.within_transform` (the same alternating projections `fixest::demean` runs) applied to the sorted long frame before the `(unit, period)` reshape, with the treatment indicator synthesized as a column since it is derived from cohorts x positional periods rather than supplied. The Frisch-Waugh-Lovell solve is `diff_diff.linalg.solve_ols(..., weights=, rank_deficient_action="silent")`: on a rank-deficient design it fits the maximal independent set, sets the aliased coefficients to `NaN` (R-style) and returns the residual computed from the identified ones, so the module reads the dropped column names off the `NaN` positions and uses the returned residual directly. This replaced a bespoke pivoted QR whose docstring claimed to drop "later columns first"; it did not, being the same norm-pivoted QR `solve_ols` uses, and dropping the same column. +- **Note (diff-diff adds standardized differences):** `covariate_balance(standardize=True)` appends `unweighted_std_diff` / `weighted_std_diff` (difference divided by the pooled SD). R does not emit these; they are additive, so parity is asserted on the R columns only. A zero pooled SD yields NaN rather than an infinity. +- **Note (balance is requested up front, not bolted on):** R mutates a `decomposed_twfe` object in a second pass (`twfe_cov_bal`). diff-diff computes the table at construction when `balance_covariates=` is supplied and exposes it via `covariate_balance()`, so the result never retains the raw panel — consistent with the `AggregationKit` data-minimization contract. Calling `covariate_balance()` without having requested it raises with the fix inlined. + +### R output parity + +Goldens: `benchmarks/data/twfeweights_golden.json`, plus two simulated sibling panel CSVs. The `mpdta` fixture reads the shared `benchmarks/data/mpdta_stata_panel.csv` and derives `lpop_t` from a `derived_columns` expression in the golden, rather than committing a renamed copy; the generator asserts the two sources agree to CSV round-trip precision. Every cells block carries ORIGINAL period labels: `implicit_*` run in positional time internally, and the generator maps them back so one convention holds throughout and the tests assert labels rather than array position. Regenerated by `benchmarks/R/generate_twfeweights_golden.R`; R is needed only to regenerate, never to run the tests. Tests: `tests/test_twfe_weights_parity.py`. + +Three fixtures: `mpdta` (real; non-`1..T` period labels; provenance `data(mpdta, package="did")`), `sim_staggered` (three equal cohorts of 100, which is exactly what makes the comparison-group normalizer vanish at `t=3`, so this fixture deliberately exercises the degenerate cells above; its `pre_period_contribution` is non-zero but the ~0.093 is sampling noise, since `x1` is iid and cohorts are assigned by unit index, so no differential pre-trend is designed in), and `unbalanced_cohorts` (120/70/60, which breaks the `p_g == 1/3` degeneracy that would let a cohort-share bug pass silently on the equal-cohort fixture). + +| Surface | Gate | Rationale | +|---------|------|-----------| +| ATT(g,t) weights, all three aggregations | `atol=1e-12` | Closed-form rational expression in cohort masses; only double-precision representation error separates the two sides. Observed max deviation 4.7e-16. | +| `implied_att` (R's own ATT(g,t) fed back in) | `atol=1e-12` | Isolates the weight arithmetic from CallawaySantAnna-vs-`did` parity. | +| End-to-end from a CS fit | `rtol=1e-6` | COMPOSED check — carries the pre-existing CS parity band, not this module's. | +| FWL decomposition scalars and cell weights | `atol=1e-10` | R double-demeans with `fixest::demean`, iterative alternating projections at a 1e-8 fixed-point tolerance; ours is the exact closed form on a balanced panel. The gap is fixest's convergence slack. | +| FWL with covariates | `atol=1e-8` | The demeaning slack propagates through the OLS projection of `Ddot` on `Xdot`. | +| Covariate balance (11 statistics) | `atol=1e-9` | Smooth functions of the weights above. Observed max deviation 7.3e-11. | +| Per-cell ATT, `pre_period_contribution` / `post_only`, and (under `gmin1`) the decomposition/remainder split at DEGENERATE cells | `atol=5e-2` | R reports 0/0 rounding noise there; we report the exact limit. Degeneracy is DETECTED from the weight structure, never hard-coded to a fixture or period, and `estimate` stays on the tight gate everywhere. Under `base_period="first_period"` the remainder is identically zero, so the split is gated tight even where the mask fires. | +| `effective_sample_size` and per-cell `ess` at DEGENERATE cells | expected value rebuilt from R's own cells | R's `ess` at a 0/0 cell is a ratio of rounding errors (scalar gap ~0.99, per-cell up to ~0.53). The expectation uses R's weights and R's `ess` wherever the degeneracy is not detected, substituting our limit value only at the detected cells, so the assertion is anchored to R rather than to our own implementation. | + +--- + ## HonestDiD **Primary source:** [Rambachan, A., & Roth, J. (2023). A More Credible Approach to Parallel Trends. *Review of Economic Studies*, 90(5), 2555-2591.](https://doi.org/10.1093/restud/rdad018) diff --git a/docs/references.rst b/docs/references.rst index 29c162e93..e9f76999d 100644 --- a/docs/references.rst +++ b/docs/references.rst @@ -292,6 +292,11 @@ Multi-Period and Staggered Adoption - **Baker, A., Callaway, B., Cunningham, S., Goodman-Bacon, A., & Sant'Anna, P. H. C. (2025).** "Difference-in-Differences Designs: A Practitioner's Guide." *arXiv preprint* arXiv:2503.13323. https://arxiv.org/abs/2503.13323 + Primary source for the implicit-TWFE-weight diagnostics + (:func:`diff_diff.attgt_weights`, :func:`diff_diff.decompose_twfe_weights`). + Reference implementation: the ``twfeweights`` R package (v0.9.0) by Brantly + Callaway, MIT License, Copyright (c) 2023 Brantly Callaway. + Source for the 8-step practitioner workflow surfaced via ``diff_diff.get_llm_guide("practitioner")`` and the README ``## Practitioner Workflow`` section. See ``docs/methodology/REGISTRY.md`` for the diff-diff renumbering and per-step deviations. Double/Debiased Machine Learning diff --git a/tests/helpers/results_foundation.py b/tests/helpers/results_foundation.py index 09b3dbff7..3b7251ff3 100644 --- a/tests/helpers/results_foundation.py +++ b/tests/helpers/results_foundation.py @@ -43,6 +43,48 @@ def make_constructed_diagnostics() -> Dict[str, Any]: poly = pd.DataFrame({"rdplot_x": [-1.0, 0.0, 1.0], "rdplot_y": [0.9, 1.4, 2.1]}) coef = pd.DataFrame({"side": ["left", "right"], "coef_0": [1.0, 2.0]}) + # TWFE weight diagnostics: a 2-cohort x 2-period grid with one negative + # weight, so summary() exercises the negative-weight branch. + attgt_weight_cells = pd.DataFrame( + { + "group": [2, 2, 3, 3], + "time": [2, 3, 2, 3], + "post": [1, 1, 0, 1], + "weight": [0.6, 0.5, -0.2, 0.1], + "att": [1.0, 1.2, 0.0, 0.8], + } + ) + decomposition_cells = pd.DataFrame( + { + "group": [2, 2, 3, 3], + "time": [2, 3, 2, 3], + "post": [1, 1, 0, 1], + "att": [1.0, 1.2, 0.0, 0.8], + "weight": [0.4, 0.3, 0.1, 0.2], + "ess": [8.0, 8.0, 6.0, 6.0], + "remainder": [0.0, 0.0, 0.0, 0.0], + } + ) + decomposition_balance = pd.DataFrame( + { + "group": [2, 2, 3, 3], + "time": [2, 3, 2, 3], + "post": [1, 1, 0, 1], + "covariate": ["x1", "x1", "x1", "x1"], + "unweighted_treated": [0.5, 0.5, 0.4, 0.4], + "unweighted_control": [0.3, 0.3, 0.2, 0.2], + "unweighted_diff": [0.2, 0.2, 0.2, 0.2], + "weighted_treated": [0.5, 0.5, 0.4, 0.4], + "weighted_control": [0.45, 0.45, 0.38, 0.38], + "weighted_diff": [0.05, 0.05, 0.02, 0.02], + "sd": [1.0, 1.0, 1.0, 1.0], + "unweighted_log_ratio_sd": [0.01, 0.01, 0.02, 0.02], + "weighted_log_ratio_sd": [0.005, 0.005, 0.01, 0.01], + "unweighted_frac_extreme": [0.05, 0.05, 0.06, 0.06], + "weighted_frac_extreme": [0.04, 0.04, 0.05, 0.05], + } + ) + qug = diff_diff.QUGTestResults( t_stat=1.2, p_value=0.23, @@ -290,5 +332,60 @@ def make_constructed_diagnostics() -> Dict[str, Any]: interpretation="All applicable checks passed.", applicable_checks=("parallel_trends",), ), + # Every count / share is DERIVED from the cells so the fixture cannot + # drift from the object it imitates. + "ATTGTWeightsResult": diff_diff.ATTGTWeightsResult( + weights=attgt_weight_cells, + aggregation="twfe", + implied_att=float((attgt_weight_cells["weight"] * attgt_weight_cells["att"]).sum()), + n_negative=int((attgt_weight_cells["weight"] < 0).sum()), + negative_weight_share=float( + attgt_weight_cells["weight"].clip(upper=0).abs().sum() + / attgt_weight_cells["weight"].abs().sum() + ), + n_negative_post=int( + ((attgt_weight_cells["weight"] < 0) & (attgt_weight_cells["post"] == 1)).sum() + ), + negative_post_weight_share=float( + attgt_weight_cells.loc[attgt_weight_cells["post"] == 1, "weight"] + .clip(upper=0) + .abs() + .sum() + / attgt_weight_cells.loc[attgt_weight_cells["post"] == 1, "weight"].abs().sum() + ), + n_cells=len(attgt_weight_cells), + source="CallawaySantAnnaResults", + control_group="never_treated", + base_period="universal", + ), + "TWFEDecompositionResult": diff_diff.TWFEDecompositionResult( + cells=decomposition_cells, + method="fwl", + estimate=float((decomposition_cells["weight"] * decomposition_cells["att"]).sum()), + decomposition=float((decomposition_cells["weight"] * decomposition_cells["att"]).sum()), + remainder=0.0, + pre_period_contribution=float( + (decomposition_cells["weight"] * decomposition_cells["att"])[ + decomposition_cells["post"] == 0 + ].sum() + ), + post_only=float( + (decomposition_cells["weight"] * decomposition_cells["att"])[ + decomposition_cells["post"] == 1 + ].sum() + ), + base_period="first_period", + covariates=("x1",), + # Module identity: post_count * sum_post(weight * ess). + effective_sample_size=float( + (decomposition_cells["post"] == 1).sum() + * (decomposition_cells["weight"] * decomposition_cells["ess"])[ + decomposition_cells["post"] == 1 + ].sum() + ), + n_units=12, + n_periods=4, + balance=decomposition_balance, + ), } return instances diff --git a/tests/test_diagnostic_marker.py b/tests/test_diagnostic_marker.py index c390abcec..df758e1a5 100644 --- a/tests/test_diagnostic_marker.py +++ b/tests/test_diagnostic_marker.py @@ -49,6 +49,8 @@ "HADPretestReport", "DiagnosticReportResults", "DurationDiDPretestResults", + "ATTGTWeightsResult", + "TWFEDecompositionResult", ] # Representative ESTIMATOR results: marked with BaseResults, never Diagnostic. diff --git a/tests/test_doc_snippets.py b/tests/test_doc_snippets.py index 79586aff3..45a32cf4e 100644 --- a/tests/test_doc_snippets.py +++ b/tests/test_doc_snippets.py @@ -47,6 +47,7 @@ "api/dml_did.rst", "api/mmm.rst", "api/triple_diff.rst", + "api/twfe_weights.rst", "practitioner_decision_tree.rst", "practitioner_getting_started.rst", "python_comparison.rst", diff --git a/tests/test_naming_guard.py b/tests/test_naming_guard.py index c018814bf..953c371c6 100644 --- a/tests/test_naming_guard.py +++ b/tests/test_naming_guard.py @@ -565,6 +565,31 @@ def _build_rowed_index(): "TripleDifference.fit[group]": ( "rule-3 reserved treated-group 0/1 indicator (v4-design section 8 rule 3)" ), + # The TWFE weight diagnostics share vocabulary with three rename families + # without reading any of them. `time=` here is the panel PERIOD COLUMN + # NAME (the same role as CallawaySantAnna.fit[time], which no row touches), + # not the two-period 0/1 post dummy M-030/M-031/M-082/M-137/M-138 rename to + # `post`: both functions are staggered-only. `aggregation=` selects an + # ESTIMAND ("twfe" / "overall" / "simple"), not the Wooldridge output + # granularity M-044 renames to `level` and M-087 removes. + **{ + f"{fn}[time]": ( + "panel PERIOD COLUMN NAME (as in CallawaySantAnna.fit[time]), not " + "the two-period 0/1 post dummy renamed to `post` by " + "M-030/M-031/M-082/M-137/M-138; survives 4.0" + ) + for fn in ("attgt_weights", "decompose_twfe_weights") + }, + "attgt_weights[aggregation]": ( + "ESTIMAND selector ('twfe' / 'overall' / 'simple'), not the " + "WooldridgeDiDResults output granularity M-044 renames to `level` and " + "M-087 removes; survives 4.0" + ), + "ATTGTWeightsResult.aggregation": ( + "records which ESTIMAND's weights the result holds - the " + "attgt_weights[aggregation] value, not a Wooldridge output granularity " + "(M-044 / M-087); survives 4.0" + ), "run_placebo_test[time]": ( "OVERLOADED pass-through, redesign pending (TODO.md): forwarded as " "the calendar column to placebo_timing_test/placebo_group_test AND " @@ -989,6 +1014,23 @@ def _token_family_code_refs(tok): # field. The one file that DID name it - diff_diff/guides/llms-full.txt - # was migrated in this same diff (migrate-first rule) and remains a lane # hit only through its unrelated backticked schema key. + # The TWFE weight diagnostics document their own `time=` (panel period + # COLUMN) and `aggregation=` (estimand selector) on these two surfaces; + # neither reads a renamed name. See the SURFACE_ALLOWLIST entries for + # attgt_weights / decompose_twfe_weights. + ("time", "diff_diff/guides/llms.txt"): ( + "attgt_weights / decompose_twfe_weights document a panel PERIOD COLUMN " + "named `time`, not the two-period 0/1 post dummy renamed by " + "M-030/M-031/M-082/M-137/M-138" + ), + ("aggregation", "diff_diff/guides/llms.txt"): ( + "attgt_weights' ESTIMAND selector, not WooldridgeDiDResults' output " + "granularity (M-044 / M-087)" + ), + ("aggregation", "docs/methodology/REGISTRY.md"): ( + "the TWFE Weight Diagnostics section documents attgt_weights' ESTIMAND " + "selector, not WooldridgeDiDResults' output granularity (M-044 / M-087)" + ), ("estimator", "diff_diff/aggregation.py"): ( "AggregationResult.estimator - independent field holding a CLASS NAME" ), diff --git a/tests/test_twfe_weights.py b/tests/test_twfe_weights.py new file mode 100644 index 000000000..da87fd4b8 --- /dev/null +++ b/tests/test_twfe_weights.py @@ -0,0 +1,1401 @@ +"""Contract, guard and edge-case tests for the TWFE weight diagnostics. + +R output parity lives in ``tests/test_twfe_weights_parity.py``; this module +covers the behaviour that is ours rather than R's - the input guards, the +result-object surface, and the design restrictions we enforce as errors. +""" + +import numpy as np +import pandas as pd +import pytest + +import diff_diff +from diff_diff.twfe_weights import attgt_weights + + +def _panel(seed=11, n_per_cohort=40, n_periods=5, cohorts=(0, 3, 4)): + """Balanced staggered panel with a never-treated group.""" + rng = np.random.RandomState(seed) + first_treat = np.repeat(np.array(cohorts), n_per_cohort) + n_units = len(first_treat) + unit_fe = rng.normal(size=n_units) + rows = [] + for t in range(1, n_periods + 1): + treated = (first_treat != 0) & (t >= first_treat) + rows.append( + pd.DataFrame( + { + "unit": np.arange(n_units), + "period": t, + "first_treat": first_treat, + "outcome": ( + unit_fe + + 0.5 * t + + 1.0 * treated * (t - first_treat + 1) + + rng.normal(scale=0.3, size=n_units) + ), + } + ) + ) + return pd.concat(rows, ignore_index=True).sort_values(["unit", "period"]) + + +def _fit(df, **kwargs): + params = {"control_group": "never_treated", "base_period": "universal"} + params.update(kwargs) + return diff_diff.CallawaySantAnna(**params).fit( + df, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + + +@pytest.fixture(scope="module") +def panel(): + return _panel() + + +@pytest.fixture(scope="module") +def fitted(panel): + return _fit(panel) + + +class TestPublicSurface: + def test_exported_from_package_root(self): + assert diff_diff.attgt_weights is attgt_weights + for name in ( + "attgt_weights", + "decompose_twfe_weights", + "ATTGTWeightsResult", + "TWFEDecompositionResult", + "plot_twfe_weights", + ): + assert name in diff_diff.__all__, name + assert hasattr(diff_diff, name), name + + def test_name_is_distinct_from_the_dcdh_surface(self): + """The two weight surfaces must stay separately addressable.""" + assert diff_diff.attgt_weights is not diff_diff.twowayfeweights + assert diff_diff.ATTGTWeightsResult is not diff_diff.TWFEWeightsResult + + def test_result_is_a_diagnostic_without_the_quintet(self, fitted): + result = attgt_weights(fitted) + assert isinstance(result, diff_diff.Diagnostic) + for banned in ("att", "se", "t_stat", "p_value", "conf_int"): + assert not hasattr(result, banned) + + def test_result_renders(self, fitted): + result = attgt_weights(fitted) + text = result.summary() + assert "Implicit Weights on ATT(g, t)" in text + assert "TWFE regression" in text + frame = result.to_dataframe() + assert list(frame.columns) == ["group", "time", "post", "weight", "att"] + # to_dataframe hands back a copy, not the live table + frame.loc[0, "weight"] = 999.0 + assert result.weights.loc[0, "weight"] != 999.0 + assert set(result.to_dict()) >= {"aggregation", "implied_att", "weights"} + assert "aggregation='twfe'" in repr(result) + + +class TestAggregationBehaviour: + @pytest.mark.parametrize("aggregation", ["overall", "simple"]) + def test_target_estimands_are_convex(self, fitted, aggregation): + """ATT^O / ATT^simple weights are non-negative and sum to one.""" + weights = attgt_weights(fitted, aggregation=aggregation).weights["weight"] + assert (weights >= 0).all() + assert weights.sum() == pytest.approx(1.0, abs=1e-12) + + def test_twfe_weights_can_be_negative(self, fitted): + """The whole point of the diagnostic: staggered TWFE is not convex.""" + result = attgt_weights(fitted, aggregation="twfe") + assert result.n_negative > 0 + assert 0.0 < result.negative_weight_share < 1.0 + assert "Negative POST-period cells:" in result.summary() + assert result.n_negative_post <= result.n_negative + + def test_pre_treatment_cells_carry_weight_under_twfe(self, fitted): + """TWFE loads on pre-treatment cells; the CS estimands do not.""" + twfe = attgt_weights(fitted, aggregation="twfe").weights + assert (twfe.loc[twfe["post"] == 0, "weight"].abs() > 0).any() + for aggregation in ("overall", "simple"): + benign = attgt_weights(fitted, aggregation=aggregation).weights + assert (benign.loc[benign["post"] == 0, "weight"] == 0).all() + + def test_rejects_unknown_aggregation(self, fitted): + with pytest.raises(ValueError, match="aggregation must be one of"): + attgt_weights(fitted, aggregation="everything") + + +class TestDesignGuards: + def test_rejects_non_universal_base_period_for_twfe(self, panel): + fit = _fit(panel, base_period="varying") + with pytest.raises(ValueError, match="base_period='universal'"): + attgt_weights(fit, aggregation="twfe") + + def test_varying_base_is_fine_for_the_cs_estimands(self, panel): + """Only the TWFE formula needs the complete grid.""" + fit = _fit(panel, base_period="varying") + for aggregation in ("overall", "simple"): + result = attgt_weights(fit, aggregation=aggregation) + assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + + def test_rejects_not_yet_treated_control_for_twfe(self, panel): + fit = _fit(panel, control_group="not_yet_treated") + with pytest.raises(ValueError, match="control_group='never_treated'"): + attgt_weights(fit, aggregation="twfe") + + def test_rejects_repeated_cross_sections(self, panel): + # A true RCS needs one observation per unit id, so re-key the rows + # rather than just flipping the flag (panel=False rejects duplicates). + rcs = panel.copy().reset_index(drop=True) + rcs["unit"] = np.arange(len(rcs)) + fit = _fit(rcs, panel=False) + with pytest.raises(ValueError, match="requires a panel fit"): + attgt_weights(fit) + + +class TestDataFrameFallback: + def test_requires_the_full_panel_spec(self, fitted, panel): + frame = fitted.to_dataframe("group_time") + with pytest.raises(ValueError, match="missing"): + attgt_weights(frame, data=panel, unit="unit") + + def test_rejects_panel_args_alongside_a_fitted_result(self, fitted, panel): + with pytest.raises(ValueError, match="only for the DataFrame fallback"): + attgt_weights( + fitted, + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_accepts_an_att_column_as_well_as_effect(self, fitted, panel): + frame = fitted.to_dataframe("group_time")[["group", "time", "effect"]] + via_effect = attgt_weights( + frame, data=panel, unit="unit", time="period", first_treat="first_treat" + ) + via_att = attgt_weights( + frame.rename(columns={"effect": "att"}), + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + np.testing.assert_allclose( + via_effect.weights["weight"], via_att.weights["weight"], atol=1e-15 + ) + + def test_rejects_a_frame_without_an_effect_column(self, panel): + frame = pd.DataFrame({"group": [3], "time": [3]}) + with pytest.raises(ValueError, match="'effect' or 'att'"): + attgt_weights( + frame, + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_rejects_time_varying_cohort_labels(self, fitted, panel): + broken = panel.copy() + broken.loc[broken.index[0], "first_treat"] = 99 + with pytest.raises(ValueError, match="varies within unit"): + attgt_weights( + fitted.to_dataframe("group_time"), + data=broken, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_source_is_recorded(self, fitted, panel): + assert attgt_weights(fitted).source == "CallawaySantAnnaResults" + from_frame = attgt_weights( + fitted.to_dataframe("group_time"), + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + assert from_frame.source == "DataFrame" + + +class TestNonConsecutiveTimeLabels: + """Positional rescaling: gapped period labels must not change the weights.""" + + def test_gapped_periods_match_consecutive_ones(self, panel): + consecutive = attgt_weights(_fit(panel), aggregation="twfe") + + gapped = panel.copy() + remap = {1: 10, 2: 20, 3: 30, 4: 40, 5: 50} + gapped["period"] = gapped["period"].map(remap) + gapped["first_treat"] = gapped["first_treat"].map(lambda g: remap.get(g, 0)) + result = attgt_weights(_fit(gapped), aggregation="twfe") + + np.testing.assert_allclose( + result.weights["weight"].to_numpy(), + consecutive.weights["weight"].to_numpy(), + atol=1e-14, + ) + assert result.implied_att == pytest.approx(consecutive.implied_att, abs=1e-14) + + +class TestSamplingWeights: + def test_uniform_weights_are_a_no_op(self, fitted, panel): + baseline = attgt_weights( + fitted.to_dataframe("group_time"), + aggregation="overall", + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + weighted_panel = panel.assign(w=1.0) + weighted = attgt_weights( + fitted.to_dataframe("group_time"), + aggregation="overall", + data=weighted_panel, + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + np.testing.assert_allclose( + baseline.weights["weight"], weighted.weights["weight"], atol=1e-15 + ) + + def test_reweighting_a_cohort_shifts_its_weight(self, fitted, panel): + """Doubling a cohort's sampling weight raises its share of ATT^O.""" + baseline = attgt_weights( + fitted.to_dataframe("group_time"), + aggregation="overall", + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + tilted_panel = panel.assign(w=np.where(panel["first_treat"] == 3, 2.0, 1.0)) + tilted = attgt_weights( + fitted.to_dataframe("group_time"), + aggregation="overall", + data=tilted_panel, + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + mass_3_before = baseline.weights.query("group == 3")["weight"].sum() + mass_3_after = tilted.weights.query("group == 3")["weight"].sum() + assert mass_3_after > mass_3_before + assert tilted.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + + def test_rejects_time_varying_sampling_weights(self, fitted, panel): + broken = panel.copy() + broken["w"] = np.arange(len(broken), dtype=float) + with pytest.raises(ValueError, match="must be time-invariant"): + attgt_weights( + fitted.to_dataframe("group_time"), + data=broken, + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + + def test_rejects_a_weights_column_name_on_the_fitted_path(self, fitted): + with pytest.raises(ValueError, match="only name a column"): + attgt_weights(fitted, weights="w") + + +class TestDegenerateInputs: + def test_rejects_a_panel_with_no_treated_units(self, panel): + frame = pd.DataFrame({"group": [3.0], "time": [3.0], "effect": [1.0]}) + never = panel.assign(first_treat=0) + with pytest.raises(ValueError, match="no ever-treated units"): + attgt_weights( + frame, + data=never, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_rejects_a_frame_with_no_finite_effects(self, panel): + frame = pd.DataFrame({"group": [3.0, 3.0], "time": [3.0, 4.0], "effect": [np.nan, np.nan]}) + with pytest.raises(ValueError, match="no finite effects"): + attgt_weights( + frame, + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + + def test_warns_and_renormalizes_when_cells_are_dropped(self, fitted, panel): + frame = fitted.to_dataframe("group_time").copy() + frame.loc[frame.index[0], "effect"] = np.nan + with pytest.warns(UserWarning, match="had no estimable ATT"): + result = attgt_weights( + frame, + aggregation="overall", + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + ) + assert result.n_dropped_cells == 1 + assert len(result.weights) == len(frame) - 1 + assert "Non-estimable cells dropped:" in result.summary() + + def test_rejects_a_cohort_label_off_the_period_grid(self, panel): + frame = pd.DataFrame({"group": [3.0], "time": [3.0], "effect": [1.0]}) + broken = panel.copy() + broken.loc[broken["first_treat"] == 4, "first_treat"] = 99 + with pytest.raises(ValueError, match="not one of the observed time periods"): + attgt_weights( + frame, + data=broken, + unit="unit", + time="period", + first_treat="first_treat", + ) + + +class TestWeightedRegressionPin: + """Frozen-numbers pin for the WEIGHTED branches. + + No parity fixture passes ``weights=`` and R ``twfe_weights`` has no ``w=``, + so the weighted code paths (weighted two-way demeaning, weighted FWL solve, + weighted cohort masses) have no external oracle. These literals were + captured from the implementation BEFORE the linear algebra was routed + through ``diff_diff.linalg.solve_ols`` / ``diff_diff.utils.within_transform`` + and pin that behaviour: any refactor must leave them green at 1e-12. + """ + + @staticmethod + def _weighted_panel(): + rng = np.random.default_rng(20260907) + n_per, n_periods = 12, 5 + cohorts = [0] * n_per + [3] * n_per + [4] * n_per + rows = [] + for i, g in enumerate(cohorts): + w = float(rng.choice([0.5, 1.0, 1.5, 2.5])) + alpha = rng.normal() + for t in range(1, n_periods + 1): + x = rng.normal() + 0.3 * t + effect = 1.0 * (t - g + 1) if (g and t >= g) else 0.0 + y = alpha + 0.2 * t + 0.5 * x + effect + rng.normal(scale=0.3) + rows.append({"id": i, "t": t, "g": g, "y": y, "x": x, "w": w}) + return pd.DataFrame(rows) + + _DEC = { + "nocov": dict( + kwargs={}, + estimate=1.4221735897240102, + pre_period_contribution=0.5127724996498023, + post_only=0.9094010900742079, + ess=59.999999999999986, + weight=[ + -0.26383763837638374, + -0.26383763837638374, + 0.3726937269372693, + 0.07749077490774903, + 0.07749077490774903, + -0.059040590405904064, + -0.059040590405904064, + -0.3542435424354244, + 0.23616236162361626, + 0.23616236162361626, + ], + att=[ + 0.0, + -1.4436560985156102, + 0.2605332041282682, + 1.5423748763069138, + 2.020457826078349, + 0.0, + -0.8817595960499504, + -0.22533107108782402, + 1.0544057969164409, + 1.2161310006314463, + ], + ), + "cov": dict( + kwargs={"covariates": ["x"]}, + estimate=1.3930847792561663, + pre_period_contribution=0.5397671816632773, + post_only=0.8533175975928889, + ess=58.816020983744, + weight=[ + -0.26642873140338846, + -0.26217358219812914, + 0.3731356281678529, + 0.07703988099301252, + 0.07842680444065203, + -0.05851961726129342, + -0.05862391113360511, + -0.35425415800358373, + 0.23433448557874187, + 0.23706320081974044, + ], + att=[ + 0.0, + -1.4333600946359863, + 0.1470326297519402, + 1.502553090163623, + 1.9646825183908976, + 0.0, + -0.8405505765908995, + -0.32378354968013, + 1.0394622134176261, + 1.2023475510925157, + ], + ), + "gmin1": dict( + kwargs={"base_period": "gmin1"}, + estimate=1.4221735897240106, + pre_period_contribution=-0.3554385674608017, + post_only=1.777612157184812, + ess=59.999999999999986, + weight=None, # identical to nocov (weights do not depend on the base period) + att=[ + 1.4436560985156102, + 0.0, + 1.7041893026438784, + 2.986030974822525, + 3.4641139245939594, + 0.2253310710878238, + -0.6564285249621264, + 0.0, + 1.279736868004265, + 1.4414620717192705, + ], + ), + } + + @pytest.mark.parametrize("key", ["nocov", "cov", "gmin1"]) + def test_decomposition_weighted_branches(self, key): + spec = self._DEC[key] + df = self._weighted_panel() + result = diff_diff.decompose_twfe_weights( + df, outcome="y", unit="id", time="t", first_treat="g", weights="w", **spec["kwargs"] + ) + assert result.estimate == pytest.approx(spec["estimate"], abs=1e-12) + assert result.pre_period_contribution == pytest.approx( + spec["pre_period_contribution"], abs=1e-12 + ) + assert result.post_only == pytest.approx(spec["post_only"], abs=1e-12) + assert result.effective_sample_size == pytest.approx(spec["ess"], abs=1e-9) + expected_w = spec["weight"] if spec["weight"] is not None else self._DEC["nocov"]["weight"] + np.testing.assert_allclose(result.cells["weight"].to_numpy(), expected_w, atol=1e-12) + np.testing.assert_allclose(result.cells["att"].to_numpy(), spec["att"], atol=1e-12) + + _AGG = { + "twfe": ( + 1.4348838554104435, + [ + -0.2638376383763837, + -0.2638376383763837, + 0.3726937269372694, + 0.0774907749077491, + 0.0774907749077491, + -0.059040590405904085, + -0.059040590405904085, + -0.3542435424354242, + 0.2361623616236162, + 0.2361623616236162, + ], + ), + "overall": ( + 2.101822892918353, + [ + 0, + 0, + 0.17874396135265702, + 0.17874396135265702, + 0.17874396135265702, + 0, + 0, + 0, + 0.2318840579710145, + 0.2318840579710145, + ], + ), + "simple": ( + 2.239531384413449, + [ + 0, + 0, + 0.21142857142857147, + 0.21142857142857147, + 0.21142857142857147, + 0, + 0, + 0, + 0.18285714285714288, + 0.18285714285714288, + ], + ), + } + + @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) + def test_attgt_weighted_branches(self, aggregation): + df = self._weighted_panel() + cs = diff_diff.CallawaySantAnna(base_period="universal", control_group="never_treated").fit( + df, outcome="y", unit="id", time="t", first_treat="g" + ) + unit_w = df.groupby("id", sort=True)["w"].first().to_numpy() + result = attgt_weights(cs, aggregation=aggregation, weights=unit_w) + implied, weight = self._AGG[aggregation] + assert result.implied_att == pytest.approx(implied, abs=1e-12) + np.testing.assert_allclose(result.weights["weight"].to_numpy(), weight, atol=1e-12) + assert list(zip(result.weights["group"], result.weights["time"])) == [ + (3, 1), + (3, 2), + (3, 3), + (3, 4), + (3, 5), + (4, 1), + (4, 2), + (4, 3), + (4, 4), + (4, 5), + ] + + +# --------------------------------------------------------------------------- +# Review-response regression tests (PR #812 items 1-5, 9, 17, 19) +# --------------------------------------------------------------------------- + + +def _gt_frame(fitted): + return fitted.to_dataframe("group_time") + + +def _frame_call(frame, panel, **kw): + return attgt_weights( + frame, data=panel, unit="unit", time="period", first_treat="first_treat", **kw + ) + + +class TestCohortLabelValidation: + """Item 1: NaN / -inf labels are errors, never a silent never-treated unit.""" + + @pytest.mark.parametrize("bad", [np.nan, -np.inf]) + def test_decompose_rejects_non_finite_labels(self, panel, bad): + df = panel.copy() + df["first_treat"] = df["first_treat"].astype(float) + df.loc[df["unit"] == 45, "first_treat"] = bad + with pytest.raises(ValueError, match="NaN or -inf cohort label"): + diff_diff.decompose_twfe_weights( + df, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + + @pytest.mark.parametrize("bad", [np.nan, -np.inf]) + def test_frame_path_rejects_non_finite_labels(self, fitted, panel, bad): + df = panel.copy() + df["first_treat"] = df["first_treat"].astype(float) + df.loc[df["unit"] == 45, "first_treat"] = bad + with pytest.raises(ValueError, match="NaN or -inf cohort label"): + _frame_call(_gt_frame(fitted), df) + + def test_plus_inf_is_never_treated(self, fitted, panel): + df = panel.copy() + df["first_treat"] = df["first_treat"].astype(float) + df.loc[df["first_treat"] == 0, "first_treat"] = np.inf + with_inf = _frame_call(_gt_frame(fitted), df) + with_zero = _frame_call(_gt_frame(fitted), panel) + np.testing.assert_allclose( + with_inf.weights["weight"].to_numpy(), + with_zero.weights["weight"].to_numpy(), + atol=1e-15, + ) + + def test_nan_in_one_period_fails_invariance(self, fitted, panel): + df = panel.copy() + df["first_treat"] = df["first_treat"].astype(float) + df.loc[(df["unit"] == 45) & (df["period"] == 2), "first_treat"] = np.nan + with pytest.raises(ValueError, match="varies within unit"): + _frame_call(_gt_frame(fitted), df) + with pytest.raises(ValueError, match="varies within unit"): + diff_diff.decompose_twfe_weights( + df, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + + def test_nan_period_label_is_rejected_up_front(self, fitted, panel): + df = panel.copy() + df["period"] = df["period"].astype(float) + df.loc[(df["unit"] == 45) & (df["period"] == 2), "period"] = np.nan + with pytest.raises(ValueError, match="non-finite or non-numeric period"): + _frame_call(_gt_frame(fitted), df) + with pytest.raises(ValueError, match="non-finite or non-numeric period"): + diff_diff.decompose_twfe_weights( + df, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + + +class TestBalanceNaNPropagation: + """Item 2: frac_extreme's NA for <3 distinct values survives the summary roll-up.""" + + @pytest.fixture(scope="class") + def decomposed(self): + df = _panel() + rng = np.random.RandomState(3) + df["binary"] = rng.binomial(1, 0.4, size=len(df)).astype(float) + df["const"] = 1.0 + df["cont"] = rng.normal(size=len(df)) + # Make the binary / constant columns unit-invariant so the unit mean + # keeps them at <3 distinct values. + df["binary"] = df.groupby("unit")["binary"].transform("first") + return diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["cont"], + balance_covariates=["binary", "const", "cont"], + ) + + def test_cell_level_is_nan_for_degenerate_covariates(self, decomposed): + cells = decomposed.covariate_balance(level="cell") + for cov in ("binary", "const"): + sub = cells[cells["covariate"] == cov] + assert sub["unweighted_frac_extreme"].isna().all() + assert sub["weighted_frac_extreme"].isna().all() + cont = cells[cells["covariate"] == "cont"] + assert np.isfinite(cont["unweighted_frac_extreme"]).all() + + def test_summary_level_propagates_nan_not_zero(self, decomposed): + summary = decomposed.covariate_balance(level="summary").set_index("covariate") + for cov in ("binary", "const"): + assert np.isnan(summary.loc[cov, "unweighted_frac_extreme"]) + assert np.isnan(summary.loc[cov, "weighted_frac_extreme"]) + # The other statistics are ordinary sums and stay finite. + assert np.isfinite(summary.loc[cov, "unweighted_diff"]) + assert np.isfinite(summary.loc["cont", "weighted_frac_extreme"]) + + +class TestCovariateGuard: + """Item 3: a covariate-adjusted CS fit is not a TWFE regression.""" + + def test_covariate_adjusted_fit_is_rejected_under_twfe(self, panel): + df = panel.copy() + df["x"] = np.random.RandomState(5).normal(size=len(df)) + fit = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal" + ).fit( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["x"], + ) + assert fit._aggregation_kit.bookkeeping["covariates"] == ("x",) + with pytest.raises(ValueError, match="requires a fit without covariates"): + attgt_weights(fit, aggregation="twfe") + # The CS estimands do not depend on the regression specification. + assert attgt_weights(fit, aggregation="overall").n_negative_post == 0 + + def test_unadjusted_fit_records_empty_covariates(self, fitted): + assert fitted._aggregation_kit.bookkeeping["covariates"] == () + + def test_legacy_kit_without_the_key_warns(self, fitted): + kit = fitted._aggregation_kit + saved = kit.bookkeeping.pop("covariates") + try: + with pytest.warns(UserWarning, match="predates covariate bookkeeping"): + attgt_weights(fitted, aggregation="twfe") + finally: + kit.bookkeeping["covariates"] = saved + + def test_wrong_result_type_is_a_type_error(self, panel): + dec = diff_diff.decompose_twfe_weights( + panel, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + with pytest.raises(TypeError, match="CallawaySantAnna"): + attgt_weights(dec) # type: ignore[arg-type] + + +class TestFrameValidationAndGrid: + """Item 4: duplicates, non-finite cells, and incomplete grids fail closed.""" + + def test_duplicate_cells_are_rejected(self, fitted, panel): + frame = _gt_frame(fitted) + dup = pd.concat([frame, frame.iloc[[2]]], ignore_index=True) + with pytest.raises(ValueError, match="duplicated \\(group, time\\)"): + _frame_call(dup, panel) + + def test_non_finite_group_label_is_rejected(self, fitted, panel): + frame = _gt_frame(fitted) + frame.loc[0, "group"] = np.nan + with pytest.raises(ValueError, match="NaN or -inf cohort label"): + _frame_call(frame, panel) + + @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) + def test_inf_att_on_a_post_cell_is_an_incomplete_grid(self, fitted, panel, aggregation): + frame = _gt_frame(fitted) + idx = frame.index[(frame["group"] == 3) & (frame["time"] == 3)][0] + frame.loc[idx, "effect"] = np.inf + with pytest.raises(ValueError, match="complete .* grid"): + _frame_call(frame, panel, aggregation=aggregation) + + @pytest.mark.parametrize("aggregation", ["twfe", "overall", "simple"]) + def test_missing_post_cell_raises_for_every_aggregation(self, fitted, panel, aggregation): + frame = _gt_frame(fitted) + frame = frame[~((frame["group"] == 3) & (frame["time"] == 3))] + with pytest.raises(ValueError, match="required cell\\(s\\) are missing"): + _frame_call(frame, panel, aggregation=aggregation) + + def test_missing_pre_cell_raises_for_twfe_but_warns_for_cs_estimands(self, fitted, panel): + frame = _gt_frame(fitted) + frame = frame[~((frame["group"] == 4) & (frame["time"] == 1))] + with pytest.raises(ValueError, match="complete cohort x period grid"): + _frame_call(frame, panel, aggregation="twfe") + complete = _frame_call(_gt_frame(fitted), panel, aggregation="overall") + for aggregation in ("overall", "simple"): + # An ABSENT pre row is not a drop: nothing to warn about, weights unchanged. + partial = _frame_call(frame, panel, aggregation=aggregation) + assert partial.n_dropped_cells == 0 + assert len(partial.weights) == len(frame) + ref = _frame_call(_gt_frame(fitted), panel, aggregation=aggregation) + assert partial.implied_att == pytest.approx(ref.implied_att, abs=1e-15) + # A NaN pre cell (present but non-estimable) is what n_dropped_cells counts. + frame = _gt_frame(fitted) + frame.loc[frame.index[(frame["group"] == 4) & (frame["time"] == 1)][0], "effect"] = np.nan + with pytest.warns(UserWarning, match="pre-treatment group-time cell"): + dropped = _frame_call(frame, panel, aggregation="overall") + assert dropped.n_dropped_cells == 1 + assert dropped.implied_att == pytest.approx(complete.implied_att, abs=1e-15) + + def test_first_period_cohort_is_dropped_like_r_did(self): + df = _panel(cohorts=(0, 1, 3, 4), n_periods=5) + fit = _fit(df) + with pytest.warns(UserWarning, match="no estimable post-treatment cell"): + result = attgt_weights(fit, aggregation="overall") + assert 1 not in set(result.weights["group"]) + assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + # Same numbers as fitting on the panel with those units removed up front. + pre_filtered = _fit(df[df["first_treat"] != 1]) + reference = attgt_weights(pre_filtered, aggregation="overall") + np.testing.assert_allclose( + result.weights["weight"].to_numpy(), reference.weights["weight"].to_numpy(), atol=1e-12 + ) + + def test_bare_frame_missing_a_whole_non_first_cohort_raises(self, fitted, panel): + frame = _gt_frame(fitted).drop(columns=["skip_reason"], errors="ignore") + frame = frame[frame["group"] != 4] + with pytest.raises(ValueError, match="no post-treatment cell in the ATT"): + _frame_call(frame, panel, aggregation="overall") + + def test_not_yet_treated_carve_out_mirrors_aggte(self): + df = _panel(cohorts=(3, 4, 5), n_periods=6) + fit = _fit(df, control_group="not_yet_treated") + with pytest.warns(UserWarning) as record: + result = attgt_weights(fit, aggregation="overall") + messages = " | ".join(str(w.message) for w in record) + assert "structurally absent" in messages # (3,5),(3,6),(4,5),(4,6) + assert "no estimable post-treatment cell" in messages # cohort 5 + assert set(result.weights["group"]) == {3, 4} + assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-12) + post = result.weights[result.weights["post"] == 1] + # cohort 3 keeps (3,3),(3,4): divisor 2; cohort 4 keeps (4,4): divisor 1 + assert post[post["group"] == 3]["time"].tolist() == [3, 4] + assert post[post["group"] == 4]["time"].tolist() == [4] + w3 = post[post["group"] == 3]["weight"].to_numpy() + w4 = post[post["group"] == 4]["weight"].to_numpy() + assert w3[0] == pytest.approx(w3[1]) + assert w4[0] == pytest.approx(2 * w3[0]) # equal cohorts: pbar_3 == pbar_4 + expected = float((post["weight"] * post["att"]).sum()) + assert result.implied_att == pytest.approx(expected, abs=1e-12) + with pytest.raises(ValueError, match="control_group='never_treated'"): + attgt_weights(fit, aggregation="twfe") + + +class TestWeightValidation: + """Item 5: finite, non-negative, positive treated (and control) mass.""" + + @pytest.mark.parametrize( + "mutate, match", + [ + (lambda w: np.where(np.arange(len(w)) < 50, -1.0, w), "non-negative"), + (lambda w: np.where(np.arange(len(w)) == 0, np.nan, w), "must be finite"), + (lambda w: np.where(np.arange(len(w)) == 0, np.inf, w), "must be finite"), + (lambda w: np.zeros_like(w), "sum to zero"), + ], + ) + def test_bad_unit_weights_are_rejected(self, fitted, mutate, match): + w = mutate(np.ones(len(fitted._aggregation_kit.bookkeeping["unit_cohorts"]))) + with pytest.raises(ValueError, match=match): + attgt_weights(fitted, aggregation="overall", weights=w) + + def test_zero_control_mass_only_matters_where_controls_enter(self, fitted, panel): + cohorts = np.asarray(fitted._aggregation_kit.bookkeeping["unit_cohorts"], dtype=float) + w = np.where(cohorts == 0, 0.0, 1.0) + with pytest.raises(ValueError, match="never-treated comparison group carries zero"): + attgt_weights(fitted, aggregation="twfe", weights=w) + for aggregation in ("overall", "simple"): + assert attgt_weights(fitted, aggregation=aggregation, weights=w).n_cells > 0 + + def test_decompose_reports_a_nan_weight_as_non_finite(self, panel): + df = panel.copy() + df["w"] = 1.0 + df.loc[(df["unit"] == 3) & (df["period"] == 2), "w"] = np.nan + with pytest.raises(ValueError, match="must be finite"): + diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + + +class TestNegativePostWeights: + """Item 9: the pathology is negative weight on POST cells.""" + + @pytest.mark.parametrize("aggregation", ["overall", "simple"]) + def test_cs_estimands_have_no_negative_post_weight(self, fitted, aggregation): + result = attgt_weights(fitted, aggregation=aggregation) + assert result.n_negative_post == 0 + assert result.negative_post_weight_share == 0.0 + + def test_twfe_reports_both_labelled(self, fitted): + result = attgt_weights(fitted, aggregation="twfe") + assert result.n_negative_post <= result.n_negative + assert 0.0 <= result.negative_post_weight_share <= 1.0 + d = result.to_dict() + assert {"n_negative_post", "negative_post_weight_share"} <= set(d) + assert "Negative POST-period cells:" in result.summary() + + +class TestWeightedTwfeExtension: + """Item 17: weighted aggregation="twfe" has no R counterpart; tie it to the decomposition.""" + + def test_weighted_twfe_weights_match_the_weighted_decomposition(self, panel): + df = panel.copy() + rng = np.random.RandomState(9) + unit_w = pd.Series( + rng.choice([0.5, 1.0, 2.0], size=df["unit"].nunique()), + index=sorted(df["unit"].unique()), + ) + df["w"] = df["unit"].map(unit_w) + fit = _fit(df) + weighted = attgt_weights(fit, aggregation="twfe", weights=unit_w.to_numpy()) + decomposed = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + weights="w", + ) + np.testing.assert_allclose( + weighted.weights["weight"].to_numpy(), decomposed.cells["weight"].to_numpy(), atol=1e-12 + ) + + +class TestHandComputedWeights: + """Item 19: assert against numbers computed OUTSIDE the module.""" + + def test_implied_att_equals_hand_computed_overall(self): + # 2 cohorts (g=2,3) x 3 periods, 10 + 30 treated units + 20 never-treated. + frame = pd.DataFrame( + { + "group": [2, 2, 2, 3, 3, 3], + "time": [1, 2, 3, 1, 2, 3], + "effect": [0.0, 1.0, 2.0, 0.0, 0.0, 4.0], + } + ) + rows = [] + for u, g in enumerate([0] * 20 + [2] * 10 + [3] * 30): + for t in (1, 2, 3): + rows.append({"unit": u, "period": t, "first_treat": g, "outcome": 0.0}) + panel = pd.DataFrame(rows) + result = _frame_call(frame, panel, aggregation="overall") + # pbar_2 = 10/40, pbar_3 = 30/40; cohort 2 has 2 post periods, cohort 3 has 1. + expected = (10 / 40) / 2 * (1.0 + 2.0) + (30 / 40) / 1 * 4.0 + assert result.implied_att == pytest.approx(expected, abs=1e-15) + assert result.weights["weight"].sum() == pytest.approx(1.0, abs=1e-15) + + def test_all_five_public_names_are_exported(self): + for name in ( + "attgt_weights", + "decompose_twfe_weights", + "ATTGTWeightsResult", + "TWFEDecompositionResult", + "plot_twfe_weights", + ): + assert name in diff_diff.__all__, name + assert hasattr(diff_diff, name) + + +class TestCollinearCovariates: + """Item 7: rank-deficient designs go through solve_ols' R-style NaN handling.""" + + def test_exactly_collinear_pair_warns_once_and_leaves_the_estimate_unchanged(self, panel): + df = panel.copy() + rng = np.random.RandomState(21) + df["x1"] = rng.normal(size=len(df)) + df["x2"] = 2.0 * df["x1"] # exactly collinear twin + common = dict(outcome="outcome", unit="unit", time="period", first_treat="first_treat") + with pytest.warns(UserWarning, match="dropped collinear covariate") as record: + both = diff_diff.decompose_twfe_weights(df, covariates=["x1", "x2"], **common) + collinear = [w for w in record if "dropped collinear" in str(w.message)] + assert len(collinear) == 1 + message = str(collinear[0].message) + assert ("'x1'" in message) != ("'x2'" in message) # exactly one of the pair + alone = diff_diff.decompose_twfe_weights(df, covariates=["x1"], **common) + assert both.estimate == pytest.approx(alone.estimate, abs=1e-12) + np.testing.assert_allclose( + both.cells["weight"].to_numpy(), alone.cells["weight"].to_numpy(), atol=1e-12 + ) + + +class TestDecompositionEdgeCases: + """Item 8: the REGISTRY edge cases for decompose_twfe_weights, asserted.""" + + COMMON = dict(outcome="outcome", unit="unit", time="period", first_treat="first_treat") + + def test_unbalanced_panel_is_rejected(self, panel): + df = panel.drop(panel.index[(panel["unit"] == 7) & (panel["period"] == 3)]) + with pytest.raises(ValueError, match="balanced panel"): + diff_diff.decompose_twfe_weights(df, **self.COMMON) + + def test_no_never_treated_group_is_rejected(self): + df = _panel(cohorts=(3, 4)) + with pytest.raises(ValueError, match="never-treated units"): + diff_diff.decompose_twfe_weights(df, **self.COMMON) + + def test_time_varying_cohort_is_rejected(self, panel): + df = panel.copy() + df.loc[(df["unit"] == 50) & (df["period"] == 5), "first_treat"] = 4 + with pytest.raises(ValueError, match="varies within unit"): + diff_diff.decompose_twfe_weights(df, **self.COMMON) + + def test_gmin1_with_a_first_period_cohort_is_rejected(self): + df = _panel(cohorts=(0, 1, 3)) + with pytest.raises(ValueError, match="gmin1"): + diff_diff.decompose_twfe_weights(df, base_period="gmin1", **self.COMMON) + + def test_balance_without_request_and_bad_level(self, panel): + result = diff_diff.decompose_twfe_weights(panel, **self.COMMON) + with pytest.raises(ValueError, match="balance_covariates="): + result.covariate_balance() + df = panel.copy() + df["x"] = np.random.RandomState(4).normal(size=len(df)) + with_balance = diff_diff.decompose_twfe_weights(df, balance_covariates=["x"], **self.COMMON) + with pytest.raises(ValueError, match="level must be"): + with_balance.covariate_balance(level="cohort") + + def test_bad_method_and_base_period(self, panel): + with pytest.raises(ValueError, match="method must be"): + diff_diff.decompose_twfe_weights(panel, method="aipw", **self.COMMON) + with_bad = dict(self.COMMON) + with pytest.raises(ValueError, match="base_period must be"): + diff_diff.decompose_twfe_weights(panel, base_period="universal", **with_bad) + + def test_identities_hold(self, panel, fitted): + result = diff_diff.decompose_twfe_weights(panel, **self.COMMON) + assert result.estimate == pytest.approx(result.decomposition + result.remainder, abs=1e-12) + assert result.pre_period_contribution + result.post_only == pytest.approx( + result.decomposition, abs=1e-12 + ) + assert result.remainder == 0.0 + assert attgt_weights(fitted, aggregation="twfe").implied_att == pytest.approx( + result.estimate, abs=1e-6 + ) + gmin1 = diff_diff.decompose_twfe_weights(panel, base_period="gmin1", **self.COMMON) + assert gmin1.estimate == pytest.approx(result.estimate, abs=1e-10) + assert gmin1.estimate == pytest.approx(gmin1.decomposition + gmin1.remainder, abs=1e-12) + + +class TestPlotTWFEWeights: + """Item 8 / 13: matplotlib behaviour of plot_twfe_weights.""" + + @pytest.fixture(autouse=True) + def _agg_backend(self): + matplotlib = pytest.importorskip("matplotlib") + matplotlib.use("Agg") + yield + import matplotlib.pyplot as plt + + plt.close("all") + + def test_weights_view(self, fitted): + result = attgt_weights(fitted, aggregation="twfe") + ax = diff_diff.plot_twfe_weights(result, show=False) + assert ax.get_xlabel() == "Implicit weight" + # Item 9: the title counts POST-only negatives. This panel has negative + # weights, but all of them are on pre cells, so the title must not claim + # any negative treatment-period weight. + assert result.n_negative > 0 + assert result.n_negative_post == 0 + assert "negative" not in ax.get_title() + assert len(ax.collections) == 2 # post + pre scatters + + def test_weights_title_reports_post_only_count(self): + """A panel WITH negative post weight labels the post-only count.""" + fit = _fit(_panel(cohorts=(0, 2, 4))) + result = attgt_weights(fit, aggregation="twfe") + assert result.n_negative_post > 0 + ax = diff_diff.plot_twfe_weights(result, show=False) + assert f"({result.n_negative_post} negative post)" in ax.get_title() + + def test_ax_reuse_and_annotate(self, fitted): + import matplotlib.pyplot as plt + + _, ax = plt.subplots() + out = diff_diff.plot_twfe_weights( + attgt_weights(fitted, aggregation="overall"), ax=ax, annotate=True, show=False + ) + assert out is ax + assert len(ax.texts) == 10 + + def test_balance_view_and_auto(self, panel): + df = panel.copy() + df["x"] = np.random.RandomState(8).normal(size=len(df)) + dec = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["x"], + balance_covariates=["x"], + ) + ax = diff_diff.plot_twfe_weights(dec, show=False) # auto -> balance + assert "balance" in ax.get_title().lower() + ax2 = diff_diff.plot_twfe_weights(dec, kind="weights", show=False) + assert ax2.get_ylabel() == "ATT(g, t)" + + def test_balance_requested_without_table_raises(self, panel): + dec = diff_diff.decompose_twfe_weights( + panel, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + with pytest.raises(ValueError, match="no covariate balance table"): + diff_diff.plot_twfe_weights(dec, kind="balance", show=False) + + def test_all_nan_balance_table_is_a_clear_error(self, panel): + df = panel.copy() + df["const"] = 1.0 # zero pooled SD -> standardized diffs are all NaN + dec = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + balance_covariates=["const"], + ) + with pytest.raises(ValueError, match="no finite differences"): + diff_diff.plot_twfe_weights(dec, kind="balance", show=False) + + def test_bad_kind_and_backend(self, fitted): + result = attgt_weights(fitted) + with pytest.raises(ValueError, match="kind must be"): + diff_diff.plot_twfe_weights(result, kind="heat", show=False) + with pytest.raises(ValueError, match="backend must be"): + diff_diff.plot_twfe_weights(result, backend="bokeh", show=False) + + +# --------------------------------------------------------------------------- +# Second review round (PR #812): canonical time keys, anticipation, balance/ +# provenance, and an independent multi-covariate pin. +# --------------------------------------------------------------------------- + +_DECO = dict(outcome="outcome", unit="unit", time="period", first_treat="first_treat") + + +class TestCanonicalTimeKey: + """Item 1: numeric and numeric-string period labels must agree. + + ``"10" < "2"`` lexicographically, so a raw-label sort desynchronizes the + reshape from the positional grid and silently rebuilds a different panel. + """ + + @staticmethod + def _gapped(panel, labels): + df = panel.copy() + remap = {1: labels[0], 2: labels[1], 3: labels[2], 4: labels[3], 5: labels[4]} + df["period"] = df["period"].map(remap) + df["first_treat"] = df["first_treat"].map(lambda g: remap.get(g, 0)) + return df + + def test_numeric_string_labels_agree(self, panel): + as_int = diff_diff.decompose_twfe_weights(self._gapped(panel, [1, 2, 10, 11, 12]), **_DECO) + as_str = diff_diff.decompose_twfe_weights( + self._gapped(panel, ["1", "2", "10", "11", "12"]), **_DECO + ) + assert as_str.estimate == pytest.approx(as_int.estimate, abs=1e-12) + np.testing.assert_allclose( + as_str.cells["weight"].to_numpy(), as_int.cells["weight"].to_numpy(), atol=1e-12 + ) + np.testing.assert_allclose( + as_str.cells["att"].to_numpy(), as_int.cells["att"].to_numpy(), atol=1e-12 + ) + + +class TestAnticipation: + """Item 2: the anticipation window shifts the CS estimands, never TWFE.""" + + @staticmethod + def _fit(df, anticipation, control_group): + return diff_diff.CallawaySantAnna( + control_group=control_group, base_period="universal", anticipation=anticipation + ).fit(df, **_DECO) + + @staticmethod + def _group_overall(fit, panel): + group = fit.aggregate("group") + ever = panel.drop_duplicates("unit")["first_treat"].to_numpy() + ever = ever[ever != 0] + vals, counts = np.unique(ever, return_counts=True) + pg = dict(zip(vals.tolist(), (counts / counts.sum()).tolist())) + return sum(pg[lab] * att for lab, att in zip(group.label, group.att)) + + @pytest.mark.parametrize("control_group", ["never_treated", "not_yet_treated"]) + def test_simple_matches_aggregate(self, panel, control_group): + fit = self._fit(panel, 1, control_group) + got = attgt_weights(fit, aggregation="simple").implied_att + assert got == pytest.approx(fit.aggregate("simple").att[0], abs=1e-12) + + @pytest.mark.parametrize("control_group", ["never_treated", "not_yet_treated"]) + def test_overall_matches_group_combination(self, panel, control_group): + fit = self._fit(panel, 1, control_group) + got = attgt_weights(fit, aggregation="overall").implied_att + assert got == pytest.approx(self._group_overall(fit, panel), abs=1e-12) + + def test_twfe_ignores_anticipation(self, panel): + ant0 = attgt_weights(self._fit(panel, 0, "never_treated"), aggregation="twfe") + ant1 = attgt_weights(self._fit(panel, 1, "never_treated"), aggregation="twfe") + np.testing.assert_allclose( + ant1.weights["weight"].to_numpy(), ant0.weights["weight"].to_numpy(), atol=1e-12 + ) + + def test_frame_path_takes_explicit_anticipation(self, panel): + fit = self._fit(panel, 1, "never_treated") + from_fit = attgt_weights(fit, aggregation="simple") + from_frame = attgt_weights( + fit.to_dataframe("group_time"), + aggregation="simple", + data=panel, + unit="unit", + time="period", + first_treat="first_treat", + anticipation=1, + ) + assert from_frame.implied_att == pytest.approx(from_fit.implied_att, abs=1e-15) + + def test_rejects_negative_anticipation(self, fitted): + with pytest.raises(ValueError, match="non-negative"): + attgt_weights(fitted, anticipation=-1) + + def test_fitted_path_rejects_the_kwarg(self, panel): + fit = self._fit(panel, 1, "never_treated") + with pytest.raises(ValueError, match="only for the DataFrame fallback"): + attgt_weights(fit, anticipation=1) + + +class TestUnbalancedFittedResult: + """Item 3: cohort shares assume a balanced panel; reject otherwise.""" + + def test_unbalanced_fitted_result_is_rejected(self): + df = _panel() + broken = df.copy() + broken.loc[broken.index[0], "outcome"] = np.nan # row present, value missing + fit = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal" + ).fit(broken, **_DECO) + assert fit._aggregation_kit.bookkeeping["is_balanced"] is False + with pytest.raises(ValueError, match="balanced panel"): + attgt_weights(fit, aggregation="twfe") + + def test_balanced_fit_records_the_flag(self, fitted): + assert fitted._aggregation_kit.bookkeeping["is_balanced"] is True + + def test_frame_path_requires_one_obs_per_unit_period(self, fitted, panel): + broken = panel.drop(panel.index[(panel["unit"] == 7) & (panel["period"] == 3)]) + with pytest.raises(ValueError, match="exactly one observation per unit-period"): + _frame_call(_gt_frame(fitted), broken) + + def test_frame_path_rejects_duplicate_cells(self, fitted, panel): + dup = pd.concat([panel, panel.iloc[[0]]], ignore_index=True) + with pytest.raises(ValueError, match="duplicate \\(unit, period\\)"): + _frame_call(_gt_frame(fitted), dup) + + +class TestFiniteInputs: + """Item 4: NaN inputs fail closed instead of returning an all-NaN result.""" + + def test_nan_outcome_is_rejected(self, panel): + df = panel.copy() + df.loc[df.index[0], "outcome"] = np.nan + with pytest.raises(ValueError, match="outcome.*non-finite"): + diff_diff.decompose_twfe_weights(df, **_DECO) + + def test_nan_covariate_is_rejected(self, panel): + df = panel.copy() + df["x"] = 1.0 + df.loc[df.index[0], "x"] = np.inf + with pytest.raises(ValueError, match="covariate.*non-finite"): + diff_diff.decompose_twfe_weights(df, covariates=["x"], **_DECO) + + def test_nan_balance_covariate_is_rejected(self, panel): + df = panel.copy() + df["x"] = 1.0 + df.loc[df.index[0], "x"] = np.nan + with pytest.raises(ValueError, match="balance covariate.*non-finite"): + diff_diff.decompose_twfe_weights(df, balance_covariates=["x"], **_DECO) + + +class TestCarveOutConsistency: + """Item 5: the frame path honours zero_treated_control, not control_group.""" + + def test_paths_agree_on_a_not_yet_treated_fit(self): + df = _panel(cohorts=(3, 4, 5), n_periods=6) + fit = _fit(df, control_group="not_yet_treated") + with pytest.warns(UserWarning, match="structurally absent"): + from_fit = attgt_weights(fit, aggregation="overall") + with pytest.warns(UserWarning, match="structurally absent"): + from_frame = attgt_weights( + fit.to_dataframe("group_time"), + aggregation="overall", + data=df, + unit="unit", + time="period", + first_treat="first_treat", + ) + np.testing.assert_allclose( + from_fit.weights["weight"].to_numpy(), + from_frame.weights["weight"].to_numpy(), + atol=1e-15, + ) + assert from_fit.implied_att == pytest.approx(from_frame.implied_att, abs=1e-15) + + +class TestCohortDropIsStructural: + """Item 6: a mid cohort blanked out for a non-structural reason fails closed.""" + + def test_fitted_mid_cohort_without_a_reason_raises(self): + fit = _fit(_panel()) + for t in (4, 5): + cell = fit.group_time_effects[(4, t)] + cell["effect"] = np.nan + cell["skip_reason"] = None + with pytest.raises(ValueError, match="do not all carry"): + attgt_weights(fit, aggregation="overall") + + def test_frame_mid_cohort_without_a_reason_raises(self, fitted, panel): + frame = _gt_frame(fitted).copy() + mask = (frame["group"] == 4) & (frame["time"] >= 4) + frame.loc[mask, "effect"] = np.nan + frame.loc[mask, "skip_reason"] = None + with pytest.raises(ValueError, match="do not all carry"): + _frame_call(frame, panel, aggregation="overall") + + +class TestWeightsLength: + """Item 11: a wrong-length weights= is a clear error, never an IndexError.""" + + def test_wrong_length_with_an_excluded_cohort(self): + fit = _fit(_panel(cohorts=(0, 1, 3, 4))) + n_units = len(fit._aggregation_kit.bookkeeping["unit_cohorts"]) + with pytest.raises(ValueError, match="weights has length"): + attgt_weights(fit, aggregation="overall", weights=np.ones(n_units - 1)) + + +class TestMultiCovariatePin: + """Item 13: a frozen-numbers pin for the multi-column solve_ols branch. + + Two NON-collinear covariates are the first coverage of ``k > 1`` (the only + prior test used an exactly collinear pair, which exercises the rank-drop + path, not the multi-column solve). Captured from the implementation; any + refactor must leave these green at 1e-12. + """ + + @staticmethod + def _panel_two_covariates(): + rng = np.random.default_rng(20260914) + n_per, n_periods = 15, 5 + cohorts = [0] * n_per + [3] * n_per + [4] * n_per + rows = [] + for i, g in enumerate(cohorts): + alpha = rng.normal() + x1u, x2u = rng.normal(), rng.normal() + for t in range(1, n_periods + 1): + x1 = x1u + 0.4 * t + rng.normal(scale=0.1) + x2 = x2u - 0.2 * t + rng.normal(scale=0.1) + effect = 1.0 * (t - g + 1) if (g and t >= g) else 0.0 + y = alpha + 0.2 * t + 0.5 * x1 - 0.3 * x2 + effect + rng.normal(scale=0.3) + rows.append({"id": i, "t": t, "g": g, "y": y, "x1": x1, "x2": x2}) + return pd.DataFrame(rows) + + def test_two_covariate_decomposition(self): + df = self._panel_two_covariates() + result = diff_diff.decompose_twfe_weights( + df, outcome="y", unit="id", time="t", first_treat="g", covariates=["x1", "x2"] + ) + assert result.estimate == pytest.approx(1.5221483563387275, abs=1e-12) + assert result.pre_period_contribution == pytest.approx(-0.0878362103974791, abs=1e-12) + assert result.post_only == pytest.approx(1.6099845667362065, abs=1e-12) + assert result.remainder == 0.0 + assert result.covariates == ("x1", "x2") + np.testing.assert_allclose( + result.cells["weight"].to_numpy(), + [ + -0.245653467623804, + -0.253634060114308, + 0.381887799688092, + 0.055465013979673, + 0.061934714070347, + -0.068055542075608, + -0.055903047966494, + -0.376753882219787, + 0.256467817300994, + 0.244244654960894, + ], + atol=1e-12, + ) + + +class TestSignedBalancePlot: + """Item 10: the signed balance view draws the full reference diagonal.""" + + @pytest.fixture(autouse=True) + def _agg_backend(self): + matplotlib = pytest.importorskip("matplotlib") + matplotlib.use("Agg") + yield + import matplotlib.pyplot as plt + + plt.close("all") + + @staticmethod + def _decomposed(panel): + df = panel.copy() + df["x"] = np.random.RandomState(8).normal(size=len(df)) + return diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["x"], + balance_covariates=["x"], + ) + + def test_signed_reference_line_spans_negative(self, panel): + ax = diff_diff.plot_twfe_weights( + self._decomposed(panel), kind="balance", absolute_value=False, show=False + ) + line = next(ln for ln in ax.lines if ln.get_label() == "no improvement") + assert line.get_xdata()[0] < 0 + + def test_absolute_reference_line_starts_at_zero(self, panel): + ax = diff_diff.plot_twfe_weights( + self._decomposed(panel), kind="balance", absolute_value=True, show=False + ) + line = next(ln for ln in ax.lines if ln.get_label() == "no improvement") + assert line.get_xdata()[0] == 0 diff --git a/tests/test_twfe_weights_parity.py b/tests/test_twfe_weights_parity.py new file mode 100644 index 000000000..777257912 --- /dev/null +++ b/tests/test_twfe_weights_parity.py @@ -0,0 +1,611 @@ +"""R ``twfeweights`` output-parity tests for the TWFE weight diagnostics. + +Loads pre-computed golden values from +``benchmarks/data/twfeweights_golden.json`` (generated by +``benchmarks/R/generate_twfeweights_golden.R``) and asserts that the Python +implementation matches R ``twfeweights`` 0.9.0. + +**R is only needed to regenerate the JSON**, never to run these tests. The +committed JSON plus the panel CSVs it names (two simulated siblings, plus +the shared ``mpdta_stata_panel.csv``) are the source of truth and the +assertions run on any Python-only environment. Tests skip ONLY if a fixture +file is absent. + +Tolerances are module constants with a stated rationale; see the tolerance +table in ``docs/methodology/REGISTRY.md`` under "TWFE Weight Diagnostics". +""" + +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +import diff_diff +from diff_diff.twfe_weights import attgt_weights + +DATA_DIR = Path(__file__).parents[1] / "benchmarks" / "data" +GOLDEN_PATH = DATA_DIR / "twfeweights_golden.json" +REGENERATE = "Rscript benchmarks/R/generate_twfeweights_golden.R" + +FIXTURES = ("mpdta", "sim_staggered", "unbalanced_cohorts") +AGGREGATIONS = ("twfe", "overall", "simple") + +# Closed-form weights: both sides evaluate the same rational expression in +# cohort masses in double precision, so only representation error separates +# them. Observed max deviation across all 3 fixtures x 3 aggregations is +# 4.7e-16 - two orders of margin below this gate. +WEIGHT_ATOL = 1e-12 +WEIGHT_RTOL = 0.0 + +# Composed check: our CallawaySantAnna ATT(g,t) vs R did::att_gt, then the +# weights on top. Bounded by the pre-existing CS parity band, not by anything +# this module introduces. +CS_COMPOSED_RTOL = 1e-6 + + +@pytest.fixture(scope="module") +def golden(): + """Load the committed R goldens; skip when absent.""" + if not GOLDEN_PATH.exists(): + pytest.skip(f"golden file not found at {GOLDEN_PATH}; run: {REGENERATE}") + with open(GOLDEN_PATH) as fh: + return json.load(fh) + + +def _fixture(golden, name): + """Panel for a fixture, with any golden-declared derived columns applied. + + ``fixtures.mpdta`` points at the SHARED ``mpdta_stata_panel.csv`` rather + than a renamed copy of it, and declares its one derived column + (``lpop_t``) as an expression evaluated against that file's own names. + Fixtures without a ``derived_columns`` block are unaffected. + """ + payload = golden["fixtures"][name] + path = DATA_DIR / payload["data_file"] + if not path.exists(): + pytest.skip(f"panel {path} not found; run: {REGENERATE}") + df = pd.read_csv(path) + for column, expression in (payload.get("derived_columns") or {}).items(): + df[column] = df.eval(expression) + return payload, df + + +def _sorted_golden_weights(block): + """Golden weight table, sorted to the same key order the API emits.""" + return ( + pd.DataFrame( + { + "group": block["group"], + "time": block["time"], + "post": block["post"], + "weight": block["weight"], + "att": block["att"], + } + ) + .sort_values(["group", "time"]) + .reset_index(drop=True) + ) + + +def _fit_cs(df, cols): + return diff_diff.CallawaySantAnna(control_group="never_treated", base_period="universal").fit( + df, + outcome=cols["outcome"], + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + ) + + +class TestATTGTWeightsParity: + """Weights asserted against R using R's OWN ATT(g,t) values. + + Feeding the golden ``att`` column back in isolates the weight arithmetic + from CallawaySantAnna-vs-``did`` parity, which is covered separately by + ``csdid_golden_values.json``. A regression here is a regression in THIS + module. + """ + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("aggregation", AGGREGATIONS) + def test_weight_column(self, golden, fixture, aggregation): + payload, df = _fixture(golden, fixture) + expected = _sorted_golden_weights(payload["attgt_weights"][aggregation]) + + gt_frame = expected[["group", "time", "att"]].rename(columns={"att": "effect"}) + cols = payload["columns"] + result = attgt_weights( + gt_frame, + aggregation=aggregation, + data=df, + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + ) + actual = result.weights.sort_values(["group", "time"]).reset_index(drop=True) + + assert len(actual) == len(expected) + np.testing.assert_array_equal(actual["group"].to_numpy(), expected["group"].to_numpy()) + np.testing.assert_array_equal(actual["time"].to_numpy(), expected["time"].to_numpy()) + np.testing.assert_array_equal(actual["post"].to_numpy(), expected["post"].to_numpy()) + np.testing.assert_allclose( + actual["weight"].to_numpy(), + expected["weight"].to_numpy(), + atol=WEIGHT_ATOL, + rtol=WEIGHT_RTOL, + ) + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("aggregation", AGGREGATIONS) + def test_implied_att(self, golden, fixture, aggregation): + payload, df = _fixture(golden, fixture) + block = payload["attgt_weights"][aggregation] + expected = _sorted_golden_weights(block) + cols = payload["columns"] + + result = attgt_weights( + expected[["group", "time", "att"]].rename(columns={"att": "effect"}), + aggregation=aggregation, + data=df, + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + ) + np.testing.assert_allclose( + result.implied_att, + block["implied_att"], + atol=WEIGHT_ATOL, + rtol=WEIGHT_RTOL, + ) + + +class TestATTGTWeightsFromCSFit: + """End-to-end: fit CallawaySantAnna, then weight its own ATT(g,t). + + This is a COMPOSED check - it multiplies this module's parity by + CallawaySantAnna-vs-``did`` parity. It is deliberately looser than + :class:`TestATTGTWeightsParity`, and a failure here with that class green + points at CS, not at the weights. + """ + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("aggregation", AGGREGATIONS) + def test_end_to_end(self, golden, fixture, aggregation): + payload, df = _fixture(golden, fixture) + block = payload["attgt_weights"][aggregation] + result = attgt_weights(_fit_cs(df, payload["columns"]), aggregation=aggregation) + actual = result.weights.sort_values(["group", "time"]).reset_index(drop=True) + expected = _sorted_golden_weights(block) + + np.testing.assert_allclose( + actual["weight"].to_numpy(), + expected["weight"].to_numpy(), + atol=WEIGHT_ATOL, + rtol=WEIGHT_RTOL, + ) + # ATT(g,t) come from our own fit here, so this leg carries the CS band. + np.testing.assert_allclose( + actual["att"].to_numpy(), + expected["att"].to_numpy(), + rtol=CS_COMPOSED_RTOL, + atol=1e-9, + ) + np.testing.assert_allclose( + result.implied_att, + block["implied_att"], + rtol=CS_COMPOSED_RTOL, + atol=1e-9, + ) + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_negative_weights_are_a_twfe_phenomenon(self, golden, fixture): + """TWFE puts negative weight on some cells; the CS estimands never do.""" + payload, df = _fixture(golden, fixture) + fit = _fit_cs(df, payload["columns"]) + + twfe = attgt_weights(fit, aggregation="twfe") + assert twfe.n_negative > 0 + assert twfe.negative_weight_share > 0 + + # Post-only negative fields, anchored to R's OWN golden weights rather + # than to our implementation (which would be a tautology). + block = payload["attgt_weights"]["twfe"] + g_w = np.asarray(block["weight"], dtype=float) + g_post = np.asarray(block["post"], dtype=bool) + neg_post = (g_w < 0) & g_post + abs_post = float(np.abs(g_w[g_post]).sum()) + assert twfe.n_negative_post == int(neg_post.sum()) + assert twfe.negative_post_weight_share == pytest.approx( + float(np.abs(g_w[neg_post]).sum() / abs_post), abs=1e-12 + ) + + for aggregation in ("overall", "simple"): + benign = attgt_weights(fit, aggregation=aggregation) + assert benign.n_negative == 0 + assert benign.negative_weight_share == 0.0 + assert benign.n_negative_post == 0 + assert benign.negative_post_weight_share == 0.0 + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_target_estimand_weights_sum_to_one(self, golden, fixture): + """ATT^O and ATT^simple are proper averages of the post cells.""" + payload, df = _fixture(golden, fixture) + fit = _fit_cs(df, payload["columns"]) + for aggregation in ("overall", "simple"): + weights = attgt_weights(fit, aggregation=aggregation).weights + np.testing.assert_allclose(weights["weight"].sum(), 1.0, atol=1e-12) + + +class TestCSFitAndFrameAgree: + """The DataFrame fallback reproduces the fitted-result path exactly.""" + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("aggregation", AGGREGATIONS) + def test_paths_agree(self, golden, fixture, aggregation): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + fit = _fit_cs(df, cols) + + from_fit = attgt_weights(fit, aggregation=aggregation) + from_frame = attgt_weights( + fit.to_dataframe("group_time"), + aggregation=aggregation, + data=df, + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + ) + left = from_fit.weights.sort_values(["group", "time"]).reset_index(drop=True) + right = from_frame.weights.sort_values(["group", "time"]).reset_index(drop=True) + np.testing.assert_allclose( + left["weight"].to_numpy(), right["weight"].to_numpy(), atol=1e-15 + ) + np.testing.assert_allclose(from_fit.implied_att, from_frame.implied_att, atol=1e-15) + + +# FWL decomposition. R double-demeans with `fixest::demean`, an iterative +# alternating-projections solver with a 1e-8 fixed-point tolerance; ours is +# the exact closed form on a balanced panel. The gap is fixest's convergence +# slack, which then propagates through the OLS projection of Ddot on Xdot. +DEMEAN_ATOL = 1e-10 +DEMEAN_COV_ATOL = 1e-8 +BALANCE_ATOL = 1e-9 + +# Cells whose comparison-group implicit weights are constant AND average to +# zero: ATT(g,t) there is a 0/0 limit. We return the limit (the unweighted +# contrast, exact); R divides the rounding errors and lands ~1e-4 away. The +# weights on such cells cancel exactly in the aggregate, so `estimate` is +# unaffected - which is why the scalar assertions below stay at 1e-10 while +# the per-cell gate is relaxed only where the degeneracy is DETECTED, never +# by hard-coding a fixture or period. +DEGENERATE_CELL_ATOL = 5e-2 + +# R names balance rows `mean_` (it averages each covariate over +# periods within unit first); we keep the covariate's own name. +R_BALANCE_COLUMNS = { + "unweighted_covs_treated": "unweighted_treated", + "unweighted_covs_comparison": "unweighted_control", + "unweighted_diff": "unweighted_diff", + "weighted_covs_treated": "weighted_treated", + "weighted_covs_comparison": "weighted_control", + "weighted_diff": "weighted_diff", + "sd": "sd", + "unweighted_log_ratio_sd_diff": "unweighted_log_ratio_sd", + "weighted_log_ratio_sd_diff": "weighted_log_ratio_sd", + "unweighted_frac_treated_extreme": "unweighted_frac_extreme", + "weighted_frac_treated_extreme": "weighted_frac_extreme", +} + + +def _decompose(df, cols, **kwargs): + return diff_diff.decompose_twfe_weights( + df, + outcome=cols["outcome"], + unit=cols["unit"], + time=cols["time"], + first_treat=cols["first_treat"], + **kwargs, + ) + + +def _assert_cell_labels(cells, golden_cells): + """Row-for-row (group, time, post) agreement before any value comparison.""" + for column in ("group", "time", "post"): + np.testing.assert_array_equal( + cells[column].to_numpy(dtype=float), + np.asarray(golden_cells[column], dtype=float), + err_msg=f"cell {column!r} labels differ from the golden", + ) + + +def _degenerate_mask(cells, golden_cells): + """Rows where R's ATT(g,t) is a 0/0 artifact rather than a disagreement. + + Detected from the DATA: a degenerate cell is one whose weight is exactly + offset by another cell in the same period (they cancel in the aggregate), + which is the signature of a vanishing comparison-group normalizer. + """ + weights = np.asarray(golden_cells["weight"], dtype=float) + times = np.asarray(golden_cells["time"], dtype=float) + mask = np.zeros(len(weights), dtype=bool) + for t in np.unique(times): + in_period = times == t + if in_period.sum() > 1 and abs(weights[in_period].sum()) < 1e-12: + mask |= in_period + return mask + + +class TestDecompositionParityFWL: + """R ``implicit_twfe_weights`` parity, including the no-covariate branch.""" + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_no_covariate_branch_two_ways(self, golden, fixture): + """``covariates=None`` and a time-invariant covariate must agree. + + The golden was generated with ``xformula = ~`` + because upstream cannot run ``~1`` (``fixest::demean`` segfaults on + the zero-column model matrix). Asserting BOTH Python calls against + that single golden proves the equivalence instead of assuming it. + """ + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = payload["decompose"]["fwl_nocov"] + + without = _decompose(df, cols, covariates=None) + with pytest.warns(UserWarning, match="no within-unit-and-period variation"): + with_invariant = _decompose(df, cols, covariates=[cols["invariant_cov"]]) + + np.testing.assert_allclose( + without.cells["weight"].to_numpy(), + with_invariant.cells["weight"].to_numpy(), + atol=1e-15, + ) + assert without.estimate == pytest.approx(with_invariant.estimate, abs=1e-15) + + for result in (without, with_invariant): + _assert_cell_labels(result.cells, expected["cells"]) + np.testing.assert_allclose(result.estimate, expected["estimate"], atol=DEMEAN_ATOL) + np.testing.assert_allclose( + result.cells["weight"].to_numpy(), + np.asarray(expected["cells"]["weight"]), + atol=DEMEAN_ATOL, + ) + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("key", ["fwl_nocov", "fwl_cov", "fwl_gmin1"]) + def test_scalars(self, golden, fixture, key): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = payload["decompose"][key] + kwargs = { + "fwl_nocov": {"covariates": None}, + "fwl_cov": {"covariates": [cols["varying_cov"]]}, + "fwl_gmin1": {"covariates": None, "base_period": "gmin1"}, + }[key] + atol = DEMEAN_COV_ATOL if key == "fwl_cov" else DEMEAN_ATOL + + result = _decompose(df, cols, **kwargs) + + # `estimate` is invariant to the 0/0 cells - the weights on them + # cancel - so it is gated tightly on EVERY fixture. The + # decomposition/remainder SPLIT is not invariant: under gmin1 the + # remainder is itself built from the degenerate comparison-group + # weights, so R's noise moves mass between the two halves while + # leaving their sum exact. + np.testing.assert_allclose(result.estimate, expected["estimate"], atol=atol) + + _assert_cell_labels(result.cells, expected["cells"]) + degenerate = _degenerate_mask(result.cells, expected["cells"]) + # The decomposition/remainder SPLIT only moves under gmin1 (the remainder + # is built from the degenerate comparison-group weights). Under + # first_period the remainder is identically zero, so the split is gated + # tight even where the mask fires (observed gap on sim_staggered/fwl_nocov + # is 3e-15). + split_atol = DEGENERATE_CELL_ATOL if (degenerate.any() and key == "fwl_gmin1") else atol + for field in ("decomposition", "remainder"): + np.testing.assert_allclose(getattr(result, field), expected[field], atol=split_atol) + + # pre_period_contribution / post_only straddle the pre/post split, so R's 0/0 noise + # at the degenerate cells (which sit on opposite sides of it on + # sim_staggered) moves each by ~1.2e-4 while their sum stays exact. + pp_atol = DEGENERATE_CELL_ATOL if degenerate.any() else atol + for field in ("pre_period_contribution", "post_only"): + np.testing.assert_allclose(getattr(result, field), expected[field], atol=pp_atol) + + # effective_sample_size: post_count * sum_post(weight * ess). At the + # degenerate cells R's ess is a ratio of rounding errors (gap ~0.99 on + # sim_staggered), so the expected value is rebuilt from R's OWN cells, + # substituting our limit ess only where R's is noise - independent of + # our implementation, and not a tautology. + g_cells = expected["cells"] + r_w = np.asarray(g_cells["weight"], dtype=float) + r_ess = np.asarray(g_cells["ess"], dtype=float) + r_post = np.asarray(g_cells["post"], dtype=bool) + our_ess = result.cells["ess"].to_numpy(dtype=float) + ess_ref = np.where(degenerate, our_ess, r_ess) + expected_ess = r_post.sum() * float((r_w[r_post] * ess_ref[r_post]).sum()) + np.testing.assert_allclose(result.effective_sample_size, expected_ess, atol=1e-6) + if not degenerate.any(): + np.testing.assert_allclose( + result.effective_sample_size, expected["effective_sample_size"], atol=1e-6 + ) + + # estimate == decomposition + remainder is an identity, not a fit + assert result.estimate == pytest.approx(result.decomposition + result.remainder, abs=1e-12) + + @pytest.mark.parametrize("fixture", FIXTURES) + @pytest.mark.parametrize("key", ["fwl_nocov", "fwl_cov", "fwl_gmin1"]) + def test_cells(self, golden, fixture, key): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = payload["decompose"][key] + kwargs = { + "fwl_nocov": {"covariates": None}, + "fwl_cov": {"covariates": [cols["varying_cov"]]}, + "fwl_gmin1": {"covariates": None, "base_period": "gmin1"}, + }[key] + atol = DEMEAN_COV_ATOL if key == "fwl_cov" else DEMEAN_ATOL + + result = _decompose(df, cols, **kwargs) + # Label alignment: the golden now carries ORIGINAL period labels in + # every block, so the row-for-row comparisons below are anchored rather + # than merely positional. + _assert_cell_labels(result.cells, expected["cells"]) + np.testing.assert_allclose( + result.cells["weight"].to_numpy(), + np.asarray(expected["cells"]["weight"]), + atol=atol, + ) + + actual_att = result.cells["att"].to_numpy() + golden_att = np.asarray(expected["cells"]["att"]) + degenerate = _degenerate_mask(result.cells, expected["cells"]) + np.testing.assert_allclose(actual_att[~degenerate], golden_att[~degenerate], atol=atol) + if degenerate.any(): + np.testing.assert_allclose( + actual_att[degenerate], + golden_att[degenerate], + atol=DEGENERATE_CELL_ATOL, + ) + + # Cell ESS: tight against R where R is a valid reference; at a 0/0 cell + # the comparison weights are all one, so the cell ESS is exactly + # n_control - an anchor independent of BOTH implementations. + ours_ess = result.cells["ess"].to_numpy(dtype=float) + theirs_ess = np.asarray(expected["cells"]["ess"], dtype=float) + np.testing.assert_allclose(ours_ess[~degenerate], theirs_ess[~degenerate], atol=atol) + n_control = int((df.groupby(cols["unit"])[cols["first_treat"]].first() == 0).sum()) + np.testing.assert_allclose(ours_ess[degenerate], float(n_control), rtol=1e-12) + + # remainder: at the degenerate cells R divides rounding errors + # (cell-ess gap up to ~0.53 on sim_staggered), so only finiteness is + # asserted there. + ours_rem = result.cells["remainder"].to_numpy(dtype=float) + theirs_rem = np.asarray(expected["cells"]["remainder"], dtype=float) + np.testing.assert_allclose(ours_rem[~degenerate], theirs_rem[~degenerate], atol=atol) + assert np.isfinite(ours_rem[degenerate]).all() + + +class TestDecompositionIsExactAtDegenerateCells: + """Where R reports 0/0 noise, we report the analytic limit.""" + + def test_limit_equals_the_unweighted_contrast(self, golden): + """sim_staggered has three equal cohorts, so E_3[D] == mean_t E_t[D]. + + The comparison-group implicit weights are then constant and average to + zero. The limit of ``resid / mean(resid)`` for a constant vector is + one, so ATT(g, 3) is the plain difference of mean outcome changes - + computable here without any of the module's machinery. + """ + payload, df = _fixture(golden, "sim_staggered") + cols = payload["columns"] + result = _decompose(df, cols, covariates=None) + + wide = df.pivot(index=cols["unit"], columns=cols["time"], values=cols["outcome"]).to_numpy() + cohorts = df.groupby(cols["unit"])[cols["first_treat"]].first().to_numpy() + change = wide[:, 2] - wide[:, 0] # base_period="first_period" + control_mean = change[cohorts == 0].mean() + + for cohort in (3, 4): + expected = change[cohorts == cohort].mean() - control_mean + actual = result.cells.query("group == @cohort and time == 3")["att"] + assert actual.iloc[0] == pytest.approx(expected, abs=1e-12) + + def test_warns_about_the_degenerate_cells(self, golden): + payload, df = _fixture(golden, "sim_staggered") + with pytest.warns(UserWarning, match="0/0 limit"): + _decompose(df, payload["columns"], covariates=None) + + +class TestBalanceParity: + """R ``twfe_cov_bal`` + ``mp_covariate_bal_summary_helper`` parity.""" + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_cell_level(self, golden, fixture): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = pd.DataFrame(payload["balance"]["fwl"]["cells"]) + expected["covariate"] = expected["covariate"].str.replace("^mean_", "", regex=True) + + result = _decompose( + df, + cols, + covariates=[cols["varying_cov"]], + balance_covariates=[cols["invariant_cov"], cols["varying_cov"]], + ) + actual = result.covariate_balance(level="cell", standardize=False) + + key = ["group", "time", "covariate"] + expected = expected.sort_values(key).reset_index(drop=True) + actual = actual.sort_values(key).reset_index(drop=True) + assert actual["covariate"].tolist() == expected["covariate"].tolist() + for column in ("group", "time", "post"): + np.testing.assert_array_equal( + actual[column].to_numpy(dtype=float), + expected[column].to_numpy(dtype=float), + err_msg=f"balance cell {column!r} labels differ from the golden", + ) + + for r_name, our_name in R_BALANCE_COLUMNS.items(): + np.testing.assert_allclose( + actual[our_name].to_numpy(dtype=float), + expected[r_name].to_numpy(dtype=float), + atol=BALANCE_ATOL, + err_msg=f"balance column {our_name!r} ({fixture})", + ) + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_summary_level(self, golden, fixture): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + expected = pd.DataFrame(payload["balance"]["fwl"]["summary"]) + expected["covariate"] = expected["covariate"].str.replace("^mean_", "", regex=True) + + result = _decompose( + df, + cols, + covariates=[cols["varying_cov"]], + balance_covariates=[cols["invariant_cov"], cols["varying_cov"]], + ) + actual = result.covariate_balance(level="summary", standardize=False) + + expected = expected.sort_values("covariate").reset_index(drop=True) + actual = actual.sort_values("covariate").reset_index(drop=True) + assert actual["covariate"].tolist() == expected["covariate"].tolist() + + r_summary = { + "unweighted_treat": "unweighted_treated", + "unweighted_untreat": "unweighted_control", + "unweighted_diff": "unweighted_diff", + "weighted_treat": "weighted_treated", + "weighted_untreat": "weighted_control", + "weighted_diff": "weighted_diff", + "sd": "sd", + "unweighted_log_ratio_sd_diff": "unweighted_log_ratio_sd", + "weighted_log_ratio_sd_diff": "weighted_log_ratio_sd", + "unweighted_frac_treated_extreme": "unweighted_frac_extreme", + "weighted_frac_treated_extreme": "weighted_frac_extreme", + } + for r_name, our_name in r_summary.items(): + np.testing.assert_allclose( + actual[our_name].to_numpy(dtype=float), + expected[r_name].to_numpy(dtype=float), + atol=BALANCE_ATOL, + err_msg=f"balance summary {our_name!r} ({fixture})", + ) + + +class TestCrossSurfaceIdentity: + """attgt_weights and decompose_twfe_weights describe the same regression.""" + + @pytest.mark.parametrize("fixture", FIXTURES) + def test_twfe_weights_reproduce_the_decomposition(self, golden, fixture): + payload, df = _fixture(golden, fixture) + cols = payload["columns"] + + weighted = attgt_weights(_fit_cs(df, cols), aggregation="twfe") + decomposed = _decompose(df, cols, covariates=None) + + assert weighted.implied_att == pytest.approx(decomposed.estimate, abs=1e-6) diff --git a/tests/test_visualization_plotly.py b/tests/test_visualization_plotly.py index d3c4d774b..309ee053e 100644 --- a/tests/test_visualization_plotly.py +++ b/tests/test_visualization_plotly.py @@ -662,3 +662,93 @@ def test_band_labels(self): results.alpha = 0.025 fig_r_frac = plot_dose_response(results, backend="plotly", show=False) assert self._band_traces(fig_r_frac)[0].name == "97.5% CI" + + +class TestPlotlyTWFEWeights: + """Plotly backend for plot_twfe_weights (both views).""" + + @staticmethod + def _panel_and_fit(): + import pandas as pd + + import diff_diff + + rng = np.random.RandomState(11) + first_treat = np.repeat(np.array([0, 3, 4]), 30) + rows = [] + for t in range(1, 6): + treated = (first_treat != 0) & (t >= first_treat) + rows.append( + pd.DataFrame( + { + "unit": np.arange(len(first_treat)), + "period": t, + "first_treat": first_treat, + "outcome": rng.normal(size=len(first_treat)) + treated * 1.0, + "x": rng.normal(size=len(first_treat)), + } + ) + ) + df = pd.concat(rows, ignore_index=True) + fit = diff_diff.CallawaySantAnna( + control_group="never_treated", base_period="universal" + ).fit(df, outcome="outcome", unit="unit", time="period", first_treat="first_treat") + return df, fit + + def test_weights_view(self): + import diff_diff + + _, fit = self._panel_and_fit() + fig = diff_diff.plot_twfe_weights( + diff_diff.attgt_weights(fit), backend="plotly", show=False + ) + assert isinstance(fig, go.Figure) + assert len(fig.data) == 2 # post + pre traces + assert len(fig.layout.shapes) >= 2 # zero lines + + def test_balance_view(self): + import diff_diff + + df, _ = self._panel_and_fit() + dec = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["x"], + balance_covariates=["x"], + ) + fig = diff_diff.plot_twfe_weights(dec, backend="plotly", show=False) + assert isinstance(fig, go.Figure) + assert any(trace.name == "no improvement" for trace in fig.data) + + def test_signed_balance_reference_line_spans_negative(self): + import diff_diff + + df, _ = self._panel_and_fit() + dec = diff_diff.decompose_twfe_weights( + df, + outcome="outcome", + unit="unit", + time="period", + first_treat="first_treat", + covariates=["x"], + balance_covariates=["x"], + ) + fig = diff_diff.plot_twfe_weights( + dec, kind="balance", absolute_value=False, backend="plotly", show=False + ) + line = next(trace for trace in fig.data if trace.name == "no improvement") + assert min(line.x) < 0 + + def test_hover_labels_do_not_need_annotation(self): + """Item 18: `text` feeds the hover template even when it is not drawn.""" + import diff_diff + + _, fit = self._panel_and_fit() + fig = diff_diff.plot_twfe_weights( + diff_diff.attgt_weights(fit), backend="plotly", annotate=False, show=False + ) + assert fig.data[0].mode == "markers" + assert len(fig.data[0].text) > 0