51 Python requests Library - Comprehensive Crash Course
51.1 Quick Reference Card
"""
requests Quick Reference
========================
BASIC REQUESTS
--------------
requests.get(url, params=dict)
requests.post(url, data=dict) # Form data
requests.post(url, json=dict) # JSON data
requests.put(url, data=dict)
requests.patch(url, data=dict)
requests.delete(url)
COMMON PARAMETERS
-----------------
headers=dict # Custom headers
params=dict # Query parameters
json=dict # JSON payload
data=dict # Form data
files={'name': file_object} # File upload
auth=('user', 'pass') # Basic auth
timeout=seconds # Timeout
verify=True/False # SSL verification
proxies=dict # Proxy config
stream=True/False # Stream response
RESPONSE ATTRIBUTES
-------------------
response.status_code # HTTP status code
response.ok # True if status < 400
response.text # Response as string
response.content # Response as bytes
response.json() # Parse JSON response
response.headers # Response headers
response.cookies # Response cookies
response.url # Final URL
response.encoding # Response encoding
response.raise_for_status() # Raise HTTPError if bad status
SESSION USAGE
-------------
session = requests.Session()
session.headers.update(dict)
session.get(url)
session.close()
# Or with context manager
with requests.Session() as session:
session.get(url)
ERROR HANDLING
--------------
from requests.exceptions import RequestException, Timeout, ConnectionError
try:
response = requests.get(url, timeout=5)
response.raise_for_status()
except Timeout:
# Handle timeout
except ConnectionError:
# Handle connection error
except RequestException as e:
# Handle any request error
"""51.2 Introduction & Installation
The requests library is the de-facto standard for making HTTP requests in Python. It’s much more elegant than the built-in urllib.
# Install requests
pip install requestsimport requests
# Check version
print(requests.__version__)51.3 HTTP Basics
Understanding HTTP request-response cycle:
Client (Your Code) Server (API/Website)
| |
| 1. HTTP Request (GET/POST/etc.) |
|-------------------------------------->|
| |
| 2. Process Request |
| [Server Logic]
| |
| 3. HTTP Response (Status + Data) |
|<--------------------------------------|
| |Common HTTP Methods:
- GET: Retrieve data
- POST: Submit data
- PUT: Update/replace data
- PATCH: Partially update data
- DELETE: Remove data
- HEAD: Get headers only
- OPTIONS: Get supported methods
51.4 Making GET Requests
51.4.1 Basic GET Request
import requests
def basic_get_example():
"""Demonstrate basic GET request."""
url = "https://api.github.com/users/python"
response = requests.get(url)
print(f"Status Code: {response.status_code}")
print(f"Content: {response.json()}")
basic_get_example()51.4.2 GET with Query Parameters
def get_with_params():
"""GET request with query parameters."""
url = "https://api.github.com/search/repositories"
# Method 1: Using params dictionary (recommended)
params = {
'q': 'python requests',
'sort': 'stars',
'order': 'desc',
'per_page': 5
}
response = requests.get(url, params=params)
# Method 2: Direct URL (not recommended)
# response = requests.get("https://api.github.com/search/repositories?q=python&sort=stars")
print(f"Request URL: {response.url}")
print(f"Total Count: {response.json()['total_count']}")
get_with_params()51.5 Making POST Requests
51.5.1 POST with JSON Data
def post_json_example():
"""POST request with JSON payload."""
url = "https://httpbin.org/post"
data = {
'patient_id': 'PT123456',
'modality': 'CT',
'study_date': '2025-11-15'
}
response = requests.post(url, json=data)
print(f"Status: {response.status_code}")
print(f"Response: {response.json()['json']}")
post_json_example()51.5.2 POST with Form Data
def post_form_example():
"""POST request with form-encoded data."""
url = "https://httpbin.org/post"
# Form data (like HTML form submission)
form_data = {
'username': 'radiologist',
'department': 'radiology'
}
response = requests.post(url, data=form_data)
print(f"Form data sent: {response.json()['form']}")
post_form_example()51.5.3 File Upload
def upload_file_example():
"""Upload files via POST request."""
url = "https://httpbin.org/post"
# Single file upload
files = {'file': open('sample.txt', 'rb')}
response = requests.post(url, files=files)
# Multiple files
files = {
'file1': open('image1.jpg', 'rb'),
'file2': open('image2.jpg', 'rb')
}
response = requests.post(url, files=files)
# With additional form data
files = {'dicom': open('scan.dcm', 'rb')}
data = {'patient_id': 'PT123'}
response = requests.post(url, files=files, data=data)
# upload_file_example() # Uncomment if you have files51.6 Request Parameters & Headers
51.6.1 Custom Headers
def custom_headers_example():
"""Send custom headers with request."""
url = "https://api.github.com/users/python"
headers = {
'User-Agent': 'RadiologyApp/1.0',
'Accept': 'application/json',
'Authorization': 'Bearer YOUR_TOKEN_HERE',
'Custom-Header': 'custom-value'
}
response = requests.get(url, headers=headers)
print(f"Request headers sent: {response.request.headers}")
custom_headers_example()51.6.2 Common Header Patterns
def header_patterns():
"""Common header configurations."""
# JSON API headers
json_headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
# Authentication headers
auth_headers = {
'Authorization': 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...',
# Or Basic Auth
# 'Authorization': 'Basic dXNlcm5hbWU6cGFzc3dvcmQ='
}
# Custom API key
api_key_headers = {
'X-API-Key': 'your-api-key-here',
'X-API-Secret': 'your-secret-here'
}
header_patterns()51.7 Response Objects
51.7.1 Response Attributes & Methods
def response_exploration():
"""Explore response object attributes."""
response = requests.get("https://api.github.com/users/python")
# Status
print(f"Status Code: {response.status_code}")
print(f"OK? {response.ok}") # True if status < 400
print(f"Reason: {response.reason}")
# Headers
print(f"\nContent-Type: {response.headers['Content-Type']}")
print(f"All headers: {dict(response.headers)}")
# Content
print(f"\nRaw bytes: {response.content[:50]}")
print(f"Text: {response.text[:100]}")
print(f"JSON: {response.json()['login']}")
# Request info
print(f"\nRequest URL: {response.url}")
print(f"Request method: {response.request.method}")
# Encoding
print(f"\nEncoding: {response.encoding}")
# Cookies
print(f"Cookies: {response.cookies}")
response_exploration()51.7.2 Content Type Handling
def handle_different_content_types():
"""Handle different response content types."""
# JSON response
json_response = requests.get("https://api.github.com/users/python")
data = json_response.json()
# Plain text
text_response = requests.get("https://httpbin.org/robots.txt")
text = text_response.text
# Binary content (image, pdf, etc.)
image_response = requests.get("https://httpbin.org/image/png")
binary_data = image_response.content
# Save binary content to file
with open('downloaded_image.png', 'wb') as f:
f.write(binary_data)
# Stream large files
large_file_url = "https://httpbin.org/image/png"
with requests.get(large_file_url, stream=True) as r:
with open('streamed_file.png', 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)
# handle_different_content_types() # Uncomment to run51.9 Error Handling & Timeouts
51.9.1 Exception Handling
def error_handling_example():
"""Comprehensive error handling."""
from requests.exceptions import (
RequestException,
Timeout,
ConnectionError,
HTTPError,
TooManyRedirects
)
url = "https://api.example.com/data"
try:
response = requests.get(url, timeout=5)
response.raise_for_status() # Raise HTTPError for bad status codes
data = response.json()
print(f"Success: {data}")
except Timeout:
print("Request timed out")
except ConnectionError:
print("Failed to connect to server")
except HTTPError as e:
print(f"HTTP error occurred: {e}")
print(f"Status code: {e.response.status_code}")
except TooManyRedirects:
print("Too many redirects")
except RequestException as e:
print(f"An error occurred: {e}")
# error_handling_example() # Will fail with connection error51.9.2 Timeout Configuration
def timeout_examples():
"""Different timeout configurations."""
# Single timeout value (applies to both connect and read)
response = requests.get("https://httpbin.org/delay/1", timeout=5)
# Separate connect and read timeouts
response = requests.get(
"https://httpbin.org/delay/1",
timeout=(3.05, 27) # (connect timeout, read timeout)
)
# No timeout (wait forever - not recommended)
# response = requests.get("https://httpbin.org/delay/10", timeout=None)
timeout_examples()51.9.3 Retry Logic
def retry_with_backoff():
"""Implement retry logic with exponential backoff."""
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
# Configure retry strategy
retry_strategy = Retry(
total=3, # Total retries
backoff_factor=1, # Wait 1, 2, 4 seconds
status_forcelist=[429, 500, 502, 503, 504],
allowed_methods=["HEAD", "GET", "OPTIONS"]
)
adapter = HTTPAdapter(max_retries=retry_strategy)
session = requests.Session()
session.mount("http://", adapter)
session.mount("https://", adapter)
try:
response = session.get("https://httpbin.org/status/500")
print(f"Status: {response.status_code}")
except Exception as e:
print(f"Failed after retries: {e}")
retry_with_backoff()51.10 Authentication
51.10.1 Basic Authentication
def basic_auth_example():
"""HTTP Basic Authentication."""
from requests.auth import HTTPBasicAuth
url = "https://httpbin.org/basic-auth/user/pass"
# Method 1: Using auth parameter
response = requests.get(url, auth=HTTPBasicAuth('user', 'pass'))
# Method 2: Tuple shorthand
response = requests.get(url, auth=('user', 'pass'))
print(f"Authenticated: {response.status_code == 200}")
basic_auth_example()51.10.2 Bearer Token Authentication
def bearer_token_auth():
"""Bearer token authentication (common for APIs)."""
url = "https://api.github.com/user"
token = "ghp_your_token_here"
headers = {
'Authorization': f'Bearer {token}'
}
response = requests.get(url, headers=headers)
print(f"Status: {response.status_code}")
# bearer_token_auth() # Requires valid token51.10.3 API Key Authentication
def api_key_auth():
"""API key authentication patterns."""
# Pattern 1: Header
headers = {'X-API-Key': 'your-api-key'}
response = requests.get("https://api.example.com/data", headers=headers)
# Pattern 2: Query parameter
params = {'api_key': 'your-api-key'}
response = requests.get("https://api.example.com/data", params=params)
# Pattern 3: Custom header name
headers = {'ApiKey': 'your-api-key'}
response = requests.get("https://api.example.com/data", headers=headers)
# api_key_auth() # Example only51.11 Advanced Features
51.11.1 Proxies
def proxy_example():
"""Use HTTP/HTTPS proxies."""
proxies = {
'http': 'http://10.10.1.10:3128',
'https': 'http://10.10.1.10:1080',
}
response = requests.get("https://httpbin.org/ip", proxies=proxies)
# SOCKS proxy (requires pysocks)
# proxies = {
# 'http': 'socks5://user:pass@host:port',
# 'https': 'socks5://user:pass@host:port'
# }
# proxy_example() # Requires proxy server51.11.2 Custom Verification
def ssl_verification():
"""SSL certificate verification options."""
# Default: Verify SSL certificate
response = requests.get("https://httpbin.org/get", verify=True)
# Disable verification (not recommended for production)
response = requests.get("https://httpbin.org/get", verify=False)
# Use custom CA bundle
response = requests.get("https://httpbin.org/get", verify='/path/to/certfile')
ssl_verification()51.11.3 Streaming Large Downloads
def stream_large_file():
"""Stream large files efficiently."""
url = "https://httpbin.org/image/png"
with requests.get(url, stream=True) as response:
response.raise_for_status()
total_size = int(response.headers.get('content-length', 0))
downloaded = 0
with open('large_file.png', 'wb') as f:
for chunk in response.iter_content(chunk_size=8192):
f.write(chunk)
downloaded += len(chunk)
# Progress tracking
if total_size > 0:
progress = (downloaded / total_size) * 100
print(f"Downloaded: {progress:.1f}%", end='\r')
print("\nDownload complete!")
# stream_large_file() # Uncomment to run51.11.4 Response Hooks
def response_hooks_example():
"""Use hooks to customize response processing."""
def print_url(response, *args, **kwargs):
"""Hook to print requested URL."""
print(f"Requested: {response.url}")
def check_status(response, *args, **kwargs):
"""Hook to validate status."""
if response.status_code != 200:
print(f"Warning: Status {response.status_code}")
hooks = {
'response': [print_url, check_status]
}
response = requests.get("https://httpbin.org/get", hooks=hooks)
response_hooks_example()51.12 Real-World Examples
51.12.1 Example 1: GitHub API Client
class GitHubClient:
"""Simple GitHub API client."""
def __init__(self, token=None):
"""Initialize with optional authentication token."""
self.base_url = "https://api.github.com"
self.session = requests.Session()
if token:
self.session.headers.update({
'Authorization': f'Bearer {token}'
})
self.session.headers.update({
'Accept': 'application/vnd.github.v3+json'
})
def get_user(self, username):
"""Get user information."""
url = f"{self.base_url}/users/{username}"
response = self.session.get(url)
response.raise_for_status()
return response.json()
def get_repos(self, username, per_page=10):
"""Get user repositories."""
url = f"{self.base_url}/users/{username}/repos"
params = {'per_page': per_page, 'sort': 'updated'}
response = self.session.get(url, params=params)
response.raise_for_status()
return response.json()
def search_repositories(self, query, limit=5):
"""Search repositories."""
url = f"{self.base_url}/search/repositories"
params = {
'q': query,
'sort': 'stars',
'order': 'desc',
'per_page': limit
}
response = self.session.get(url, params=params)
response.raise_for_status()
return response.json()['items']
def close(self):
"""Close the session."""
self.session.close()
# Usage
def github_example():
"""Demonstrate GitHub client usage."""
client = GitHubClient()
try:
# Get user info
user = client.get_user('python')
print(f"User: {user['name']}")
print(f"Followers: {user['followers']}")
# Get repositories
repos = client.get_repos('python', per_page=3)
print(f"\nTop repositories:")
for repo in repos:
print(f" - {repo['name']}: {repo['description']}")
# Search
results = client.search_repositories('machine learning python', limit=3)
print(f"\nSearch results:")
for repo in results:
print(f" - {repo['full_name']} (⭐ {repo['stargazers_count']})")
finally:
client.close()
github_example()51.12.2 Example 2: Medical Imaging API (Conceptual)
class MedicalImagingAPI:
"""Conceptual medical imaging API client."""
def __init__(self, api_key, base_url):
"""Initialize API client with authentication."""
self.base_url = base_url
self.session = requests.Session()
self.session.headers.update({
'X-API-Key': api_key,
'Content-Type': 'application/json'
})
def upload_dicom(self, filepath, patient_id, study_metadata):
"""Upload DICOM file with metadata."""
url = f"{self.base_url}/studies/upload"
with open(filepath, 'rb') as f:
files = {'dicom': f}
data = {
'patient_id': patient_id,
'metadata': study_metadata
}
response = self.session.post(url, files=files, data=data)
response.raise_for_status()
return response.json()
def get_study(self, study_id):
"""Retrieve study information."""
url = f"{self.base_url}/studies/{study_id}"
response = self.session.get(url, timeout=10)
response.raise_for_status()
return response.json()
def search_studies(self, patient_id=None, modality=None, date_from=None):
"""Search studies with filters."""
url = f"{self.base_url}/studies/search"
params = {}
if patient_id:
params['patient_id'] = patient_id
if modality:
params['modality'] = modality
if date_from:
params['date_from'] = date_from
response = self.session.get(url, params=params)
response.raise_for_status()
return response.json()
def run_ai_inference(self, study_id, model_name):
"""Run AI model inference on a study."""
url = f"{self.base_url}/inference"
payload = {
'study_id': study_id,
'model_name': model_name,
'parameters': {
'threshold': 0.5,
'batch_size': 1
}
}
response = self.session.post(url, json=payload, timeout=60)
response.raise_for_status()
return response.json()
# Usage example (conceptual)
# api = MedicalImagingAPI(api_key='your-key', base_url='https://api.hospital.com')
# studies = api.search_studies(patient_id='PT123', modality='CT')51.12.3 Example 3: RESTful API CRUD Operations
class ResourceAPI:
"""Generic REST API client for CRUD operations."""
def __init__(self, base_url, auth_token=None):
"""Initialize API client."""
self.base_url = base_url.rstrip('/')
self.session = requests.Session()
if auth_token:
self.session.headers.update({
'Authorization': f'Bearer {auth_token}'
})
def create(self, resource, data):
"""CREATE: POST request."""
url = f"{self.base_url}/{resource}"
response = self.session.post(url, json=data)
response.raise_for_status()
return response.json()
def read(self, resource, resource_id=None):
"""READ: GET request."""
url = f"{self.base_url}/{resource}"
if resource_id:
url = f"{url}/{resource_id}"
response = self.session.get(url)
response.raise_for_status()
return response.json()
def update(self, resource, resource_id, data):
"""UPDATE: PUT request."""
url = f"{self.base_url}/{resource}/{resource_id}"
response = self.session.put(url, json=data)
response.raise_for_status()
return response.json()
def partial_update(self, resource, resource_id, data):
"""PARTIAL UPDATE: PATCH request."""
url = f"{self.base_url}/{resource}/{resource_id}"
response = self.session.patch(url, json=data)
response.raise_for_status()
return response.json()
def delete(self, resource, resource_id):
"""DELETE: DELETE request."""
url = f"{self.base_url}/{resource}/{resource_id}"
response = self.session.delete(url)
response.raise_for_status()
return response.status_code == 204
# Usage
api = ResourceAPI('https://jsonplaceholder.typicode.com')
# Create
# new_post = api.create('posts', {'title': 'Test', 'body': 'Content', 'userId': 1})
# Read
post = api.read('posts', 1)
print(f"Post: {post['title']}")
# Update
# updated = api.update('posts', 1, {'title': 'Updated', 'body': 'New content', 'userId': 1})
# Delete
# deleted = api.delete('posts', 1)51.13 Best Practices & Tips
51.13.1 1. Always Use Timeouts
# ❌ Bad: No timeout (can hang forever)
response = requests.get("https://api.example.com/data")
# ✅ Good: Always specify timeout
response = requests.get("https://api.example.com/data", timeout=10)51.13.2 2. Use Sessions for Multiple Requests
# ❌ Bad: Create new connection each time
for i in range(10):
response = requests.get(f"https://api.example.com/item/{i}")
# ✅ Good: Reuse connection with session
with requests.Session() as session:
for i in range(10):
response = session.get(f"https://api.example.com/item/{i}")51.13.3 3. Handle Errors Properly
# ❌ Bad: No error handling
response = requests.get("https://api.example.com/data")
data = response.json()
# ✅ Good: Proper error handling
try:
response = requests.get("https://api.example.com/data", timeout=5)
response.raise_for_status()
data = response.json()
except requests.RequestException as e:
print(f"Error: {e}")51.13.4 4. Use Context Managers for Files
# ❌ Bad: File handle might not close
files = {'file': open('data.txt', 'rb')}
response = requests.post(url, files=files)
# ✅ Good: Automatically closes file
with open('data.txt', 'rb') as f:
files = {'file': f}
response = requests.post(url, files=files)51.13.5 5. Stream Large Files
# ❌ Bad: Load entire file into memory
response = requests.get("https://example.com/large_file.zip")
with open('file.zip', 'wb') as f:
f.write(response.content)
# ✅ Good: Stream in chunks
with requests.get("https://example.com/large_file.zip", stream=True) as r:
with open('file.zip', 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
f.write(chunk)