Coverage for pytest_recap/plugin.py: 6%

212 statements  

« prev     ^ index     » next       coverage.py v7.8.0, created at 2025-05-30 22:42 -0500

1import ast 

2import json 

3import os 

4import platform 

5import socket 

6import uuid 

7from datetime import datetime, timezone 

8from typing import Dict, Generator, List, Optional, Tuple 

9from warnings import WarningMessage 

10 

11import pytest 

12from _pytest.config import Config 

13from _pytest.config.argparsing import Parser 

14from _pytest.nodes import Item 

15from _pytest.reports import TestReport 

16from _pytest.runner import CallInfo 

17from _pytest.terminal import TerminalReporter 

18 

19from pytest_recap.cloud import upload_to_cloud 

20from pytest_recap.models import RecapEvent, RerunTestGroup, TestResult, TestSession, TestSessionStats 

21from pytest_recap.storage import JSONStorage 

22 

23import logging 

24 

25 

26# --- Global warning collection. This is required because Pytest hook pytest-warning-recorded 

27# does not pass the Config object, so it cannot be used to store warnings. 

28_collected_warnings = [] 

29 

30 

31# --- pytest hooks --- # 

32def pytest_addoption(parser: Parser) -> None: 

33 """Add command line options for pytest-recap, supporting environment variable defaults. 

34 

35 Args: 

36 parser (Parser): The pytest parser object. 

37 """ 

38 group = parser.getgroup("Pytest Recap") 

39 recap_env = os.environ.get("RECAP_ENABLE", "0").lower() 

40 recap_default = recap_env in ("1", "true", "yes", "y") 

41 group.addoption( 

42 "--recap", 

43 action="store_true", 

44 default=recap_default, 

45 help="Enable pytest-recap plugin (or set environment variable RECAP_ENABLE)", 

46 ) 

47 recap_dest_env = os.environ.get("RECAP_DESTINATION") 

48 if recap_dest_env: 

49 recap_dest_default = recap_dest_env 

50 else: 

51 timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") 

52 default_dir = os.path.expanduser("~/.pytest-recap-sessions") 

53 os.makedirs(default_dir, exist_ok=True) 

54 recap_dest_default = os.path.join(default_dir, f"{timestamp}-recap.json") 

55 group.addoption( 

56 "--recap-destination", 

57 action="store", 

58 default=recap_dest_default, 

59 help="Specify pytest-recap storage destination (filepath) (or set environment variable RECAP_DESTINATION)", 

60 ) 

61 group.addoption( 

62 "--recap-system-under-test", 

63 action="store", 

64 default=None, 

65 help="JSON or Python dict string for system under test metadata (or set RECAP_SYSTEM_UNDER_TEST)", 

66 ) 

67 group.addoption( 

68 "--recap-testing-system", 

69 action="store", 

70 default=None, 

71 help="JSON or Python dict string for testing system metadata (or set RECAP_TESTING_SYSTEM)", 

72 ) 

73 group.addoption( 

74 "--recap-session-tags", 

75 action="store", 

76 default=None, 

77 help="JSON or Python dict string for session tags (or set RECAP_SESSION_TAGS)", 

78 ) 

79 group.addoption( 

80 "--recap-pretty", 

81 action="store_true", 

82 default=None, 

83 help="Pretty-print recap JSON output (or set RECAP_PRETTY=1, or ini: recap_pretty=1)", 

84 ) 

85 

86 parser.addini("recap_system_under_test", "System under test dict (JSON or Python dict string)", default="") 

87 parser.addini("recap_testing_system", "Testing system dict (JSON or Python dict string)", default="") 

88 parser.addini("recap_session_tags", "Session tags dict (JSON or Python dict string)", default="") 

89 parser.addini("recap_pretty", "Pretty-print recap JSON output (1 for pretty, 0 for minified)", default="0") 

90 

91 

92def pytest_configure(config: Config) -> None: 

93 """Configure pytest-recap plugin. 

94 

95 Args: 

96 config (Config): The pytest Config object. 

97 """ 

98 config._recap_enabled: bool = config.getoption("--recap") 

99 config._recap_destination: str = config.getoption("--recap-destination") 

100 pretty = get_recap_option(config, "recap_pretty", "recap_pretty", "RECAP_PRETTY", default="0") 

101 config._recap_pretty: bool = str(pretty).strip().lower() in {"1", "true", "yes", "y"} 

102 

103 

104def pytest_sessionstart(session): 

105 """Reset collected warnings at the start of each test session.""" 

106 global _collected_warnings 

107 _collected_warnings = [] 

108 

109 

110@pytest.hookimpl(hookwrapper=True) 

111def pytest_runtest_makereport(item: Item, call: CallInfo) -> Generator: 

112 """Hook into pytest's test report generation to generate start and stop times, if not set already.""" 

113 outcome = yield 

114 

115 logger = logging.getLogger(__name__) 

116 

117 report: TestReport = outcome.get_result() 

118 if report.when == "setup" and not hasattr(report, "start"): 

119 logger.warning(f"Setting start time for {report.nodeid} since it was not set previously") 

120 setattr(report, "start", datetime.now(timezone.utc).timestamp()) 

121 

122 if report.when == "teardown" and not hasattr(report, "stop"): 

123 logger.warning(f"Setting stop time for {report.nodeid} since it was not set previously") 

124 setattr(report, "stop", datetime.now(timezone.utc).timestamp()) 

125 

126 

127def pytest_warning_recorded(warning_message: WarningMessage, when: str, nodeid: str, location: tuple): 

128 """Collect warnings during pytest session for recap reporting. 

129 

130 Args: 

131 warning_message (WarningMessage): The warning message object. 

132 when (str): When the warning was recorded (e.g., 'call', 'setup', etc.). 

133 nodeid (str): Node ID of the test (if any). 

134 location (tuple): Location tuple (filename, lineno, function). 

135 """ 

136 _collected_warnings.append( 

137 RecapEvent( 

138 nodeid=nodeid, 

139 when=when, 

140 message=str(warning_message.message), 

141 category=getattr(warning_message.category, "__name__", str(warning_message.category)), 

142 filename=warning_message.filename, 

143 lineno=warning_message.lineno, 

144 location=location, 

145 ) 

146 ) 

147 

148 

149@pytest.hookimpl(hookwrapper=True) 

150def pytest_terminal_summary(terminalreporter: TerminalReporter, exitstatus: int, config: Config) -> None: 

151 """Hook into pytest's terminal summary to collect test results, errors, warnings, and write recap file. 

152 

153 Args: 

154 terminalreporter (TerminalReporter): The pytest terminal reporter object. 

155 exitstatus (int): Exit status of the pytest session. 

156 config (Config): The pytest config object. 

157 """ 

158 yield 

159 

160 if not getattr(config, "_recap_enabled", False): 

161 return 

162 

163 test_results_tuple: Tuple[List[TestResult], datetime, datetime] = collect_test_results_and_session_times( 

164 terminalreporter 

165 ) 

166 test_results, session_start, session_end = test_results_tuple 

167 rerun_groups: List[RerunTestGroup] = build_rerun_groups(test_results) 

168 

169 errors = [ 

170 RecapEvent( 

171 nodeid=getattr(rep, "nodeid", None), 

172 when=getattr(rep, "when", None), 

173 outcome=getattr(rep, "outcome", None), 

174 longrepr=str(getattr(rep, "longrepr", "")), 

175 sections=list(getattr(rep, "sections", [])), 

176 keywords=list(getattr(rep, "keywords", [])), 

177 # message, category, filename, lineno, location use defaults 

178 ) 

179 for rep in terminalreporter.stats.get("error", []) 

180 ] 

181 

182 warnings = _collected_warnings.copy() 

183 

184 session: TestSession = build_recap_session( 

185 test_results, session_start, session_end, rerun_groups, errors, warnings, terminalreporter, config 

186 ) 

187 # Print summary of warnings and errors using RecapEvent helpers 

188 warning_count = sum(bool(w.is_warning()) for w in warnings) 

189 error_count = sum(bool(e.is_error()) for e in errors) 

190 terminalreporter.write_sep("-", f"Recap: {warning_count} warnings, {error_count} errors collected") 

191 

192 # Optionally, print details 

193 if warning_count > 0: 

194 terminalreporter.write_line("\nWarnings:") 

195 for w in warnings: 

196 if w.is_warning(): 

197 terminalreporter.write_line(f" {w.filename}:{w.lineno} [{w.category}] {w.message}") 

198 if error_count > 0: 

199 terminalreporter.write_line("\nErrors:") 

200 for e in errors: 

201 if e.is_error(): 

202 terminalreporter.write_line(f" {e.nodeid} [{e.when}] {e.longrepr}") 

203 

204 write_recap_file(session, getattr(config, "_recap_destination", None), terminalreporter) 

205 

206 

207# --- pytest-recap-specific functions, only used internally --- # 

208def collect_test_results_and_session_times( 

209 terminalreporter: TerminalReporter, 

210) -> Tuple[List[TestResult], datetime, datetime]: 

211 """Collect test results and session times from the terminal reporter. 

212 

213 Args: 

214 terminalreporter (TerminalReporter): The terminal reporter object. 

215 

216 Returns: 

217 tuple: A tuple containing the list of test results, session start time, and session end time. 

218 

219 """ 

220 stats: Dict[str, List[TestReport]] = terminalreporter.stats 

221 test_results: List[TestResult] = [] 

222 session_start: Optional[datetime] = None 

223 session_end: Optional[datetime] = None 

224 

225 def to_dt(val: Optional[float]) -> Optional[datetime]: 

226 return datetime.fromtimestamp(val, timezone.utc) if val is not None else None 

227 

228 for outcome, reports in stats.items(): 

229 if not outcome or outcome == "warnings": 

230 continue 

231 for report in reports: 

232 if not isinstance(report, TestReport): 

233 continue 

234 if report.when == "call" or ( 

235 report.when in ("setup", "teardown") and report.outcome in ("failed", "error", "skipped") 

236 ): 

237 report_time = to_dt(getattr(report, "start", None) or getattr(report, "starttime", None)) 

238 report_end = to_dt(getattr(report, "stop", None) or getattr(report, "stoptime", None)) 

239 if session_start is None or (report_time and report_time < session_start): 

240 session_start = report_time 

241 if session_end is None or (report_end and report_end > session_end): 

242 session_end = report_end 

243 test_results.append( 

244 { 

245 "nodeid": report.nodeid, 

246 "outcome": outcome, 

247 "start_time": report_time, 

248 "stop_time": report_end, 

249 "longreprtext": str(getattr(report, "longrepr", "")), 

250 "capstdout": getattr(report, "capstdout", ""), 

251 "capstderr": getattr(report, "capstderr", ""), 

252 "caplog": getattr(report, "caplog", ""), 

253 } 

254 ) 

255 session_start = session_start or datetime.now(timezone.utc) 

256 session_end = session_end or datetime.now(timezone.utc) 

257 return test_results, session_start, session_end 

258 

259 

260def build_rerun_groups(test_results: List[TestResult]) -> List[RerunTestGroup]: 

261 """Build a list of RerunTestGroup objects from a list of test results. 

262 

263 Args: 

264 test_results (list): List of TestResult objects. 

265 

266 Returns: 

267 list: List of RerunTestGroup objects, each containing reruns for a nodeid. 

268 

269 """ 

270 test_result_objs = [ 

271 TestResult( 

272 nodeid=tr["nodeid"], 

273 outcome=tr["outcome"], 

274 longreprtext=tr["longreprtext"], 

275 start_time=tr["start_time"], 

276 stop_time=tr["stop_time"], 

277 ) 

278 for tr in test_results 

279 ] 

280 rerun_test_groups: Dict[str, RerunTestGroup] = {} 

281 for test_result in test_result_objs: 

282 if test_result.nodeid not in rerun_test_groups: 

283 rerun_test_groups[test_result.nodeid] = RerunTestGroup(nodeid=test_result.nodeid) 

284 rerun_test_groups[test_result.nodeid].add_test(test_result) 

285 return [group for group in rerun_test_groups.values() if len(group.tests) > 1] 

286 

287 

288def parse_dict_option( 

289 option_value: str, 

290 default: dict, 

291 option_name: str, 

292 terminalreporter: TerminalReporter, 

293 envvar: str = None, 

294 source: str = None, 

295) -> dict: 

296 """Parse a recap option string value into a Python dict. 

297 Supports both JSON and Python dict literal formats. 

298 Returns the provided default if parsing fails. 

299 """ 

300 if not option_value: 

301 return default 

302 try: 

303 return json.loads(option_value) 

304 except Exception: 

305 pass 

306 try: 

307 return ast.literal_eval(option_value) 

308 except Exception as e: 

309 src = f" from {source}" if source else "" 

310 env_info = f" (env var: {envvar})" if envvar else "" 

311 msg = ( 

312 f"WARNING: Invalid RECAP_{option_name.upper()} value{src}{env_info}: {option_value!r}. " 

313 f"Could not parse as dict: {e}. Using default." 

314 ) 

315 if terminalreporter: 

316 terminalreporter.write_line(msg) 

317 else: 

318 print(msg) 

319 return default 

320 

321 

322def get_recap_option(config, opt, ini, envvar, default=""): 

323 """Retrieve the raw option value for a recap option from CLI, environment variable, pytest.ini, or default. 

324 This function is responsible for determining the source (precedence order: CLI > env > ini > default), 

325 but does NOT parse the value into a dict—it always returns a string. 

326 """ 

327 cli_val = getattr(config.option, opt, None) 

328 if cli_val is not None and str(cli_val).strip() != "": 

329 return cli_val 

330 env_val = os.environ.get(envvar) 

331 if env_val is not None and str(env_val).strip() != "": 

332 return env_val 

333 ini_val = config.getini(ini) 

334 # If ini_val is a list (possible for ini options), join to string 

335 if isinstance(ini_val, list): 

336 ini_val = " ".join(str(x) for x in ini_val).strip() 

337 if ini_val is not None and str(ini_val).strip() != "": 

338 return ini_val.strip() 

339 return default 

340 

341 

342def build_recap_session( 

343 test_results: List[TestResult], 

344 session_start: datetime, 

345 session_end: datetime, 

346 rerun_groups: List[RerunTestGroup], 

347 errors: List[Dict], 

348 warnings: List[Dict], 

349 terminalreporter: TerminalReporter, 

350 config: Config, 

351) -> TestSession: 

352 """Build a TestSession object summarizing the test session. 

353 

354 Args: 

355 test_results (list): List of test result dicts. 

356 session_start (datetime): Session start time. 

357 session_end (datetime): Session end time. 

358 rerun_groups (list): List of RerunTestGroup objects. 

359 terminalreporter: Pytest terminal reporter. 

360 config: Pytest config object. 

361 

362 Returns: 

363 TestSession: The constructed test session object. 

364 

365 Notes: 

366 - session_tags, system_under_test, and testing_system can be set via CLI, env, or pytest.ini. 

367 

368 """ 

369 session_timestamp: str = session_start.strftime("%Y%m%d-%H%M%S") 

370 session_id: str = f"{session_timestamp}-{str(uuid.uuid4())[:8]}".lower() 

371 

372 # Session tags 

373 session_tags = parse_dict_option( 

374 get_recap_option(config, "recap_session_tags", "recap_session_tags", "RECAP_SESSION_TAGS"), 

375 {}, 

376 "session_tags", 

377 terminalreporter, 

378 ) 

379 if not isinstance(session_tags, dict): 

380 session_tags = {} 

381 

382 # System Under Test 

383 system_under_test = parse_dict_option( 

384 get_recap_option(config, "recap_system_under_test", "recap_system_under_test", "RECAP_SYSTEM_UNDER_TEST"), 

385 {"name": "pytest-recap"}, 

386 "system_under_test", 

387 terminalreporter, 

388 envvar="RECAP_SYSTEM_UNDER_TEST", 

389 ) 

390 if not isinstance(system_under_test, dict): 

391 system_under_test = {"name": "pytest-recap"} 

392 

393 # Testing System 

394 default_testing_system = { 

395 "hostname": socket.gethostname(), 

396 "platform": platform.platform(), 

397 "python_version": platform.python_version(), 

398 "pytest_version": pytest.__version__, 

399 "environment": os.environ.get("RECAP_ENV", "test"), 

400 } 

401 testing_system = parse_dict_option( 

402 get_recap_option(config, "recap_testing_system", "recap_testing_system", "RECAP_TESTING_SYSTEM"), 

403 default_testing_system, 

404 "testing_system", 

405 terminalreporter, 

406 envvar="RECAP_TESTING_SYSTEM", 

407 ) 

408 if not isinstance(testing_system, dict): 

409 testing_system = default_testing_system 

410 

411 # Session tags 

412 session_tags = parse_dict_option( 

413 get_recap_option(config, "recap_session_tags", "recap_session_tags", "RECAP_SESSION_TAGS"), 

414 {}, 

415 "session_tags", 

416 terminalreporter, 

417 ) 

418 if not isinstance(session_tags, dict): 

419 session_tags = {} 

420 

421 # Session stats 

422 test_result_objs: List[TestResult] = [TestResult.from_dict(tr) for tr in test_results] 

423 session_stats = TestSessionStats(test_result_objs, warnings_count=len(warnings)) 

424 

425 # Build and return session 

426 session = TestSession( 

427 session_id=session_id, 

428 session_tags=session_tags, 

429 system_under_test=system_under_test, 

430 testing_system=testing_system, 

431 session_start_time=session_start, 

432 session_stop_time=session_end, 

433 test_results=test_result_objs, 

434 rerun_test_groups=rerun_groups, 

435 errors=errors, 

436 warnings=warnings, 

437 session_stats=session_stats, 

438 ) 

439 return session 

440 

441 

442def write_recap_file(session: TestSession, destination: str, terminalreporter: TerminalReporter): 

443 """Write the recap session data to a file in JSON format. 

444 

445 Args: 

446 session (TestSession): The session recap object to write. 

447 destination (str): File or directory path for output. If None, a default location is used. 

448 terminalreporter: Pytest terminal reporter for output. 

449 

450 Raises: 

451 Exception: If writing the recap file fails. 

452 

453 """ 

454 recap_data: Dict = session.to_dict() 

455 now: datetime = datetime.now(timezone.utc) 

456 pretty: bool = getattr(getattr(terminalreporter, "config", None), "_recap_pretty", False) 

457 indent: Optional[int] = 2 if pretty else None 

458 json_bytes: bytes = json.dumps(recap_data, indent=indent).encode("utf-8") 

459 

460 # Cloud URI detection and dispatch 

461 if destination and ( 

462 destination.startswith("s3://") 

463 or destination.startswith("gs://") 

464 or destination.startswith("azure://") 

465 or destination.startswith("https://") 

466 ): 

467 try: 

468 upload_to_cloud(destination, json_bytes) 

469 filepath = destination 

470 except Exception as e: 

471 terminalreporter.write_line(f"RECAP PLUGIN ERROR (cloud upload): {e}") 

472 filepath = destination # Still print the path for test assertions 

473 else: 

474 # Determine the output file path (local) 

475 if destination: 

476 if os.path.isdir(destination) or destination.endswith("/"): 

477 os.makedirs(destination, exist_ok=True) 

478 filename = f"{now.strftime('%Y%m%d-%H%M%S')}_{getattr(session, 'system_under_test', {}).get('name', 'sut')}.json" 

479 filepath = os.path.join(destination, filename) 

480 else: 

481 filepath = destination 

482 parent_dir = os.path.dirname(filepath) 

483 if parent_dir: 

484 os.makedirs(parent_dir, exist_ok=True) 

485 else: 

486 base_dir = os.environ.get("SESSION_WRITE_BASE_DIR", os.path.expanduser("~/.pytest_recap_sessions")) 

487 base_dir = os.path.abspath(base_dir) 

488 date_dir = os.path.join(base_dir, now.strftime("%Y/%m")) 

489 os.makedirs(date_dir, exist_ok=True) 

490 filename = ( 

491 f"{now.strftime('%Y%m%d-%H%M%S')}_{getattr(session, 'system_under_test', {}).get('name', 'sut')}.json" 

492 ) 

493 filepath = os.path.join(date_dir, filename) 

494 filepath = os.path.abspath(filepath) 

495 try: 

496 storage = JSONStorage(filepath) 

497 # Pass indent to storage for pretty/minified output 

498 storage.save_single_session(recap_data, indent=indent) 

499 except Exception as e: 

500 terminalreporter.write_line(f"RECAP PLUGIN ERROR: {e}") 

501 raise 

502 

503 # Write recap file path/URI to terminal 

504 terminalreporter.write_sep("=", "pytest-recap") 

505 BLUE = "\033[34m" 

506 RESET = "\033[0m" 

507 

508 # Print cloud URI directly if applicable, else absolute file path 

509 def is_cloud_uri(uri): 

510 return isinstance(uri, str) and ( 

511 uri.startswith("s3://") or uri.startswith("gs://") or uri.startswith("azure://") 

512 ) 

513 

514 recap_uri = filepath if is_cloud_uri(filepath) else os.path.abspath(filepath) 

515 blue_path = f"Recap JSON written to: {BLUE}{recap_uri}{RESET}" 

516 terminalreporter.write_line(blue_path)