The Problem

Given a 3D mesh of a room, I need to:

1. Detect the walls, floor, and ceiling
2. Extract the room’s 2D boundary polygon
3. Handle real-world messiness: furniture, alcoves, doorways, windows, mirrors, and other scanning artifacts that cause noise and holes in the mesh.

Approach: Multi-Plane RANSAC

The foundational algorithm for this task is RANSAC (Random Sample Consensus) (Fischler and Bolles 1981). It is essentially a kind of Monte-Carlo algorithm that repeatedly samples minimal subsets of the data and keeps the model that explains the most data.

import numpy as np
from dataclasses import dataclass
from typing import Optional

@dataclass
class DetectedPlane:
    """A plane detected via RANSAC."""
    normal: np.ndarray      # Unit normal vector (3,)
    point: np.ndarray       # Point on plane (3,)
    inlier_indices: np.ndarray
    inlier_count: int

    @property
    def d(self) -> float:
        """Plane equation: n·x + d = 0, so d = -n·point."""
        return -np.dot(self.normal, self.point)

def ransac_plane(
    points: np.ndarray,
    threshold: float = 0.02,
    iterations: int = 200,
    min_inliers: int = 100
) -> Optional[DetectedPlane]:
    """
    Single-plane RANSAC with least-squares refinement.

    Args:
        points: (N, 3) point cloud.
        threshold: Distance threshold for inliers (meters).
        iterations: Number of RANSAC iterations.
        min_inliers: Minimum inliers for a valid plane.
    """
    n_points = len(points)
    if n_points < 3:
        return None

    best_inlier_count = 0
    best_inlier_mask = None
    best_normal = None
    best_point = None

    for _ in range(iterations):
        # Sample 3 random points
1        idx = np.random.choice(n_points, 3, replace=False)
        p1, p2, p3 = points[idx]

        # Compute normal via cross product
        v1 = p2 - p1
        v2 = p3 - p1
2        normal = np.cross(v1, v2)
        norm = np.linalg.norm(normal)
        if norm < 1e-8:  # Collinear points
            continue
        normal /= norm

        # Compute distances to plane
3        dists = np.abs(np.dot(points - p1, normal))
        inlier_mask = dists < threshold
        count = np.count_nonzero(inlier_mask)

        if count > best_inlier_count:
            best_inlier_count = count
            best_inlier_mask = inlier_mask
            best_normal = normal
            best_point = p1

    if best_inlier_count < min_inliers:
        return None

    # Refine with least-squares on inliers
    inlier_points = points[best_inlier_mask]
    centroid = np.mean(inlier_points, axis=0)
    centered = inlier_points - centroid
4    _, _, vh = np.linalg.svd(centered)
    refined_normal = vh[-1]

    return DetectedPlane(
        normal=refined_normal / np.linalg.norm(refined_normal),
        point=centroid,
        inlier_indices=np.where(best_inlier_mask)[0],
        inlier_count=best_inlier_count
    )

import numpy as np

def ransac_iterations_needed(inlier_ratio: float, confidence: float = 0.99) -> int:
    """Calculate iterations needed for given inlier ratio and confidence."""
    return int(np.ceil(np.log(1 - confidence) / np.log(1 - inlier_ratio**3)))

# For different inlier ratios
for w in [0.5, 0.3, 0.1]:
    n = ransac_iterations_needed(w)
    print(f"w = {w:.0%}: {n:,} iterations needed")

w = 50%: 35 iterations needed
w = 30%: 169 iterations needed
w = 10%: 4,603 iterations needed

NDT-RANSAC: Eliminating Spurious Planes

I implemented NDT-RANSAC based on Li et al. (Li et al. 2017).

NDT (Normal Distribution Transformation) fixes the spurious planes problem by pre-classifying regions of space into planar and non-planar cells. Instead of sampling 3 random points, we sample from locally planar regions.

@dataclass
class NDTCell:
    """
    A cell in the Normal Distribution Transform grid.

    Eigenvalue shape classification (sorted λ₁ ≤ λ₂ ≤ λ₃):
    - Planar: λ₁ << λ₂ ≈ λ₃ (λ₁/λ₂ ≤ threshold) → flat surface
    - Linear: λ₁ ≈ λ₂ << λ₃ (λ₂/λ₃ ≤ threshold) → edge/line
    - Spherical: λ₁ ≈ λ₂ ≈ λ₃ → noise or point feature
    """
    indices: np.ndarray       # Point indices belonging to this cell
    centroid: np.ndarray      # Mean position (3,)
    normal: np.ndarray        # Eigenvector of smallest eigenvalue
    eigenvalues: np.ndarray   # Sorted: λ₁ ≤ λ₂ ≤ λ₃
    is_planar: bool           # True if λ₁/λ₂ ≤ planarity_threshold
    is_linear: bool = False   # True if λ₂/λ₃ ≤ linearity_threshold

def compute_cell_features(points: np.ndarray) -> tuple:
    """Compute NDT features for a group of points."""
    centroid = np.mean(points, axis=0)
    centered = points - centroid

    # Covariance matrix
1    cov = np.dot(centered.T, centered) / len(points)

    # Eigendecomposition (returns sorted ascending)
2    eigenvalues, eigenvectors = np.linalg.eigh(cov)
    eigenvalues = np.abs(eigenvalues)

    # Normal = eigenvector of smallest eigenvalue
3    normal = eigenvectors[:, 0]
    normal = normal / (np.linalg.norm(normal) + 1e-10)

    # Planarity test: λ₁/λ₂ ≤ threshold
    planarity_threshold = 0.04
    if eigenvalues[1] < 1e-10:
        is_planar = False
    else:
        is_planar = (eigenvalues[0] / eigenvalues[1]) <= planarity_threshold

    return centroid, normal, eigenvalues, is_planar

# Demo with synthetic planar points
np.random.seed(42)
# Points on a plane z = 0 with small noise
planar_pts = np.column_stack([
    np.random.uniform(-1, 1, 100),
    np.random.uniform(-1, 1, 100),
    np.random.normal(0, 0.01, 100)  # Small z-noise
])

centroid, normal, eigenvalues, is_planar = compute_cell_features(planar_pts)
print(f"Eigenvalues: {eigenvalues}")
print(f"Ratio λ₁/λ₂: {eigenvalues[0]/eigenvalues[1]:.4f}")
print(f"Is planar: {is_planar}")
print(f"Normal: {normal} (should be ~[0, 0, 1])")

Eigenvalues: [9.45656815e-05 3.32523124e-01 3.58162504e-01]
Ratio λ₁/λ₂: 0.0003
Is planar: True
Normal: [ 0.00170904 -0.00359751  0.99999207] (should be ~[0, 0, 1])

The Key Insight: Better Sampling Probability

Because we sample from pre-classified planar cells, the probability of picking a good sample is dramatically improved:

Method	P(good sample)	w = 10%
Standard RANSAC		0.1% per iteration
NDT-RANSAC		10% per iteration

That’s a 100× speedup: instead of hoping 3 random points all land on the same surface, you pick one cell that’s already proven to be planar.

def irls_fit_plane(
    points: np.ndarray,
    max_iter: int = 10,
    convergence_threshold: float = 1e-6
) -> tuple[np.ndarray, np.ndarray]:
    """
    Robust plane fitting using IRLS with Welsch weight function.

    w(r) = exp(-r²/k²) where k = 2.985
    """
1    k_welsch = 2.985  # Standard tuning constant for 95% efficiency

    # Initial fit via SVD
    centroid = np.mean(points, axis=0)
    centered = points - centroid
    _, _, vh = np.linalg.svd(centered)
    normal = vh[-1]
    normal = normal / np.linalg.norm(normal)

    for iteration in range(max_iter):
        old_normal = normal.copy()

        # Compute residuals (distances to plane)
        residuals = np.abs(np.dot(centered, normal))

        # Welsch weights: exp(-r²/k²)
2        weights = np.exp(-(residuals ** 2) / (k_welsch ** 2))

        # Weighted centroid
        weighted_centroid = np.average(points, axis=0, weights=weights)
        centered = points - weighted_centroid

        # Weighted covariance
        weighted_cov = np.dot(centered.T * weights, centered) / weights.sum()

        # New normal from smallest eigenvalue
        eigenvalues, eigenvectors = np.linalg.eigh(weighted_cov)
3        normal = eigenvectors[:, 0]
        normal = normal / np.linalg.norm(normal)

        centroid = weighted_centroid

        # Check convergence
        if np.max(np.abs(normal - old_normal)) < convergence_threshold:
            break

    return normal, centroid

# Demo: plane with outliers
np.random.seed(42)
# Main plane: z ≈ 0
inliers = np.column_stack([
    np.random.uniform(-2, 2, 80),
    np.random.uniform(-2, 2, 80),
    np.random.normal(0, 0.05, 80)
])
# Outliers: points far from plane
outliers = np.column_stack([
    np.random.uniform(-2, 2, 20),
    np.random.uniform(-2, 2, 20),
    np.random.uniform(1, 3, 20)  # Way off the plane
])
points_with_outliers = np.vstack([inliers, outliers])

# Compare standard vs IRLS
standard_normal, _ = np.linalg.svd(points_with_outliers - points_with_outliers.mean(axis=0))[2][-1], None
standard_normal = standard_normal / np.linalg.norm(standard_normal)

irls_normal, irls_centroid = irls_fit_plane(points_with_outliers)

print(f"True normal: [0, 0, 1]")
print(f"Standard LS normal: {np.abs(standard_normal).round(3)}")
print(f"IRLS normal: {np.abs(irls_normal).round(3)}")

True normal: [0, 0, 1]
Standard LS normal: [0.154 0.073 0.985]
IRLS normal: [0.127 0.076 0.989]

from scipy.spatial import KDTree
from scipy.sparse import csr_matrix
from scipy.sparse.csgraph import connected_components

def split_by_connectivity(
    inlier_points: np.ndarray,
    neighbor_radius: float = 0.1,
    min_component_points: int = 50
) -> list[np.ndarray]:
    """
    Split points into spatially connected groups.

    Args:
        inlier_points: (N, 3) points on a detected plane
        neighbor_radius: Max distance for connectivity (2-3× RANSAC threshold)
        min_component_points: Filter small components as noise
    """
    n_points = len(inlier_points)
    if n_points < min_component_points:
        return [np.arange(n_points)]

    # Build KD-tree for neighbor queries
    tree = KDTree(inlier_points)

    # Find all neighbor pairs within radius
1    pairs = tree.query_pairs(r=neighbor_radius, output_type='ndarray')

    if len(pairs) == 0:
        return [np.arange(n_points)]  # No connections

    # Build sparse adjacency matrix
    row_indices = np.concatenate([pairs[:, 0], pairs[:, 1]])
    col_indices = np.concatenate([pairs[:, 1], pairs[:, 0]])
    data = np.ones(len(row_indices), dtype=np.int8)

    adjacency = csr_matrix(
        (data, (row_indices, col_indices)),
        shape=(n_points, n_points)
    )

    # Find connected components
2    n_components, labels = connected_components(
        adjacency, directed=False, return_labels=True
    )

    # Split into groups
    groups = []
    for comp_id in range(n_components):
        mask = labels == comp_id
        if mask.sum() >= min_component_points:
            groups.append(np.where(mask)[0])

    return groups if groups else [np.arange(n_points)]

# Demo: two separate clusters
np.random.seed(42)
cluster1 = np.random.randn(100, 3) * 0.1 + [0, 0, 0]
cluster2 = np.random.randn(100, 3) * 0.1 + [5, 0, 0]  # 5m away
combined = np.vstack([cluster1, cluster2])

groups = split_by_connectivity(combined, neighbor_radius=0.3, min_component_points=20)
print(f"Found {len(groups)} connected components")
for i, g in enumerate(groups):
    center = combined[g].mean(axis=0)
    print(f"  Component {i+1}: {len(g)} points, center at {center.round(2)}")

Found 2 connected components
  Component 1: 100 points, center at [ 0.01 -0.02  0.01]
  Component 2: 100 points, center at [ 5.   -0.01  0.  ]

The Real Challenge: Boundary Extraction

Detecting planes was the easy part. The difficult part was turning detected wall planes into a clean boundary polygon.

def compute_enclosure_score(
    x: float, z: float,
    wall_segments: list[tuple],
    ray_directions: list[tuple] = [(1, 0), (-1, 0), (0, 1), (0, -1)]
) -> int:
    """
    Count how many ray directions hit a wall.

    A point is "enclosed" if rays in multiple directions hit walls.
    """
    hits = 0
    for dx, dz in ray_directions:
        # Cast ray and check for wall intersections
        # (simplified - real implementation checks segment bounds)
        for seg_start, seg_end in wall_segments:
            if ray_intersects_segment(x, z, dx, dz, seg_start, seg_end):
                hits += 1
                break  # One hit per direction is enough
    return hits

def ray_intersects_segment(x, z, dx, dz, start, end):
    """Check if ray from (x,z) in direction (dx,dz) hits segment."""
    # Parametric ray-segment intersection
    x1, z1 = start
    x2, z2 = end

    # Ray: (x,z) + t*(dx,dz), t >= 0
    # Segment: (x1,z1) + s*(x2-x1,z2-z1), 0 <= s <= 1

    denom = dx * (z2 - z1) - dz * (x2 - x1)
    if abs(denom) < 1e-10:
        return False  # Parallel

    t = ((x1 - x) * (z2 - z1) - (z1 - z) * (x2 - x1)) / denom
    s = ((x1 - x) * dz - (z1 - z) * dx) / denom

    return t >= 0 and 0 <= s <= 1

Edge Detection: Beyond Planes

The same eigenvalue analysis that identifies planar cells can identify linear cells (edges). For linear cells, - points spread along one dominant direction while the perpendicular spreads are similarly small.

Edges provide information planes cannot: room corners (vertical edges where walls meet) and missing wall inference (edge exists but no plane detected → unscanned wall).

Tuning the Cell Size

The theoretical constraint from Li et al. (Li et al. 2017):

Where = sensor noise, = cell size, = planarity threshold.

The cell must be large enough that noise doesn’t dominate eigenvalue spread. Too small: noise overwhelms geometry, everything looks “spherical”. Too large: misses detail, merges distinct surfaces.

Next Steps

The CCW ordering failure points to a fundamental issue: we’re trying to construct a complex polygon from unordered corners. With the ceiling-height filter reducing noise, several alternative approaches to explore:

1. Convex hull + selective inward refinement: Start with the simplest valid boundary (convex hull of wall points), then selectively “push in” edges only where strong evidence of indentation exists.

2. Grid-based occupancy: Discretize the floor into a 2D grid, mark cells containing wall points, then trace the boundary using marching squares.

3. Constrained concave hull: Use a concave hull algorithm (alpha shapes) but constrain edges to align with detected Manhattan directions.

4. Boundary tracing: Instead of finding all corners then ordering them, start at one segment and “walk” the perimeter - always turning to the next adjacent segment.

The perfect room boundary extractor remains elusive, but each iteration reveals more about the problem structure. The key insight: start simple, add complexity only where data supports it.

Lessons Learned

1. Real-world data is messy. Furniture occludes walls, scanning artifacts create phantom planes, doorways create gaps that may or may not be intentional room boundaries.

2. NDT pre-classification is powerful. By reasoning about local geometry before sampling, we eliminate entire classes of errors.

3. Connected components matter. A mathematical plane can span disconnected physical surfaces - always check spatial connectivity.

Part 1: The Design

The process of designing and crafting a ring is slow and deliberate, which to me was the perfect chance to reflect on what this ring should symbolize.

My partner and I are avid gardeners and nature lovers, so I knew I wanted to incorporate some nature themes.I decided to go with a concept of two intertwined trees, which I felt was a fitting symbol for the way people and relationships grow.

It all started with a few rough sketches to visualize the concept, but I found it easier to flesh out the details in a computer. The modeling software I used was Blender, which I also learned on the go with this project. It is probably too much to dive into the details of designing a ring in Blender, but it is not as difficult as it may seem, and there are plenty of great tutorials.

My initial worries about the ring were around the durability of the ring - the main guideline I found was to keep the thickness of the ring at at minimum 1mm thickness throughout¹

Finding the right dimension is also essential. You can get the right dimension from this chart.

What is Lost Wax Casting?

There are several building methods one could use to make a ring - probably the most common are machining and lost wax casting. In my case, I knew that my design would be pretty intricate, so I opted for the later.

Lost wax casting is an ancient technique that can be used for creating detailed jewelry and sculptures. The idea is simple: start by carving a model out of wax (beeswax works well). Then, encase the wax in a plaster mold. Once the plaster hardens, the wax is melted away in a process called the burnout, leaving a hollow cavity to pour molten metal into.

Modern twists on this technique include using 3D printing to create the model out of resin or PLA. This allows for more intricate designs and faster production.

---

Casting the Ring

Closing Thoughts

Just like in love, there were challenges and set-backs, but with each misstep, I found a reminder of the patience and steady effort that a lasting relationship demands.

Cron Jobs vs Systemd Timers: A Quick Overview

Old time UNIX users will be familiar with cron jobs. Cron is a utility designed to automate tasks by running commands at scheduled times. These commands are typically defined in a file stored under /etc/cronbtab.

SHELL=/bin/bash
PATH=/sbin:/bin:/usr/sbin:/usr/bin
MAILTO=root

# Example of job definition:
# .---------------- minute (0 - 59)
# |  .------------- hour (0 - 23)
# |  |  .---------- day of month (1 - 31)
# |  |  |  .------- month (1 - 12) OR jan,feb,mar,apr ...
# |  |  |  |  .---- day of week (0 - 6) (Sunday=0 or 7) OR sun,mon,tue,wed,thu,fri,sat
# |  |  |  |  |
# *  *  *  *  * user-name  command to be executed

Step 1: Creating Systemd Services

Systemd services have the extension .service. All user-created systemd scripts will be stored in ~/.config/systemd/user/. Here are the contents of our systemd service for switching to the Adwaita-dark GTK theme:

dark-theme.service

[Unit]
Description=Change the GTK theme to dark mode.
After=graphical.target

[Service]
Type=oneshot
ExecStart=/bin/sh -c 'gsettings set org.gnome.desktop.interface gtk-theme Adwaita-dark'

[Install]
WantedBy=default.target

We will need to create a separate systemd service file to switch to a light GTK theme:

light-theme.service

[Unit]
Description=Switch to light GTK theme
After=graphical.target

[Service]
Type=oneshot
ExecStart=/bin/sh -c 'gsettings set org.gnome.desktop.interface gtk-theme Adwaita'

[Install]
WantedBy=default.target

Step 2: Creating Systemd Timers

Now, to create a timer, we make another set of files in the same directory with the same rootname plus the extension .timer. Here are the two files we need to create:

dark-theme.timer

[Unit]
Description=Change the GTK theme daily at a given time.

[Timer]
OnCalendar=*-*-* 16:00:00
Persistent=true

[Install]
WantedBy=timers.target

light-theme.timer

[Unit]
Description=Switch to light theme at 07:00

[Timer]
OnCalendar=*-*-* 07:00:00
Persistent=true

[Install]
WantedBy=timers.target

The OnCalendar setting specifies that this particular timer should run every day at 16:00 hrs. You can also create timers that run every other day, or on specific days of the week. For more on this, you can check out the ArchLinux Wiki.

Step 3: Enabling and Starting the Services

When we are done creating the four configuration files, we need to enable the services. It is recommended that we enable them at the user level (since requiring sudo access would be a hazzle and potential security risk).

Enabling the services:

systemctl --user enable dark-theme.service
systemctl --user enable light-theme.service
systemctl --user enable dark-theme.timer
systemctl --user enable light-theme.timer

Now the operating system will automatically change the GTK theme at the specified times. Furthermore, if we wish to manually change the theme, or test that the service works, we may do so using:

systemctl --user start light-theme.service

or:

systemctl --user start dark-theme.service

If we have any issues, we can check the logs using:

journalctl --user -u dark-theme.service

So, what else can we use this for?

A couple other applications come to mind:

• Daily data backups
• Daily data ingestion (web-scraping?)
• Time-based notifications (e.g. set reminders to take breaks, drink water, etc.)
• Scheduled system maintenance tasks

Have you implemented a similar setup or used systemd timers for something creative? Let me know if you have other interesting applications!

Motivation

I have a large collection of electronic books, which I manage using Calibre. Using Calibre’s “Extract ISBN” plugin, I am able to parse the ISBN identifier from most of my files, which then makes fetching the rest of the metadata very easy. (Below is an example of my library’s metadata.)

Thus, I have access to a very convenient and ever-growing virtual library of books, which I like to use on the go, and for exploratory research. Nevertheless, whenever I find a particularly good book, the thing that I want most, is to own a physical copy.

Enter here AbeBooks.com. Next to Amazon, and occasionally Ebay, it is my go-to site for buying cheap used textbooks. Given that I have stored the ISBN data for most of my electronic books, I would like to be able to automatically fetch pricing information for any book in my virtual library, perhaps even keeping track of changes in price over time.

However, until now, the main problem stopping me from writing a script to do this was that AbeBooks does not have a publicly available API… or at the very least, none that is explicitly documented.

REST APIs

REST, or Representational State Transfer, is a method used by the HTTP protocol to provide interoperability between servers. It is based on a request/response system, where a request is a “payload”, normally formatted as HTML, XML, or JSON, and the response can be a link to a resource, a data payload in any of the aforementioned formats, or a confirmation that some data was modified in the server.

Several common REST methods exist: GET, HEAD, POST, PUT, PATCH, DELETE, CONNECT, OPTIONS and TRACE. Among these, the two most common are GET and POST:

GET

• Used to request data from a server.
• Parameter data is stored in the URL of the query as string parameters
• Number of parameters is limited to the length that can fit in the URL
• Not secure for sensitive information. (Passwords can be easily seen)

POST

• Used to submit data to a server, and can modify server contents.
• Parameters are passed in the message body, rather than the URL.
• It has no restrictions on the number of parameters.
• Is more secure for sending sensitive information.

Exploring Network Packets

I found that inspecting the network packets for an AbeBooks search results page is simple, and yields promising results. If we open Firefox’s developer tools, under the Network tab, we can see a list of all the packets that are loaded. In particular we are interested in those that have a JSON response, highlighted in red below:

We can see that there are four POST requests, to a service called “pricingservice”, and one GET request to a “RecomendationsApi”.

If we look more closely at one of the POST requests, we can see which parameters it takes in:

ISBN! Just what we needed! Furthermore, looking at the response tab, we can see that this request returns the prices for new and used books, among other things:

Wrapping the API in Python

Now that we know a bit more about how AbeBooks works under the hood, we can start implementing our API wrapper in Python. We will need the requests module:

import requests

url = "https://www.abebooks.com/servlet/DWRestService/pricingservice"

# Search prices by ISBN
payload1 = {'action': 'getPricingDataByISBN',
           'isbn': 9781250297662,
           'container': 'pricingService-9781250297662'}

# Search prices by author and title
payload2 = {'action': 'getPricingDataForAuthorTitleStandardAddToBasket',
            'an': 'liu ken',
            'tn': 'broken stars',
            'container': 'oe-search-all'}

# Sending a request
resp = requests.post(url, data=payload1)
print(resp.status_code, resp.reason)
resp.json()

200 OK

{'errorTexts': [None],
 'errorCodes': [None],
 'success': True,
 'newExists': True,
 'usedExists': True,
 'pricingInfoForBestNew': {'bestListingid': 32081713842,
  'totalResults': 6,
  'bestPriceInPurchaseCurrencyWithCurrencySymbol': 'US$ 28.07',
  'bestPriceInSurferCurrencyWithCurrencySymbol': 'US$ 28.07',
  'shippingToDestinationPriceInPurchaseCurrencyWithCurrencySymbol': 'US$ 0.00',
  'shippingToDestinationPriceInSurferCurrencyWithCurrencySymbol': 'US$ 0.00',
  'shippingDestinationNameInSurferLanguage': 'U.S.A.',
  'vendorCountryNameInSurferLanguage': 'U.S.A.',
  'vendorId': 8304062,
  'bestPriceInPurchaseCurrencyValueOnly': '28.07',
  'bestShippingToDestinationPriceInPurchaseCurrencyValueOnly': '0.00',
  'purchaseCurrencySymbol': 'US$',
  'refinementList': None,
  'internationalEdition': False,
  'bookCondition': 'New',
  'bookDescription': 'LOCUS AWARD FINALIST FOR BEST ANTHOLOGYSixteen...',
  'freeShipping': True},
 'pricingInfoForBestUsed': {'bestListingid': 31422279270,
  'totalResults': 12,
  'bestPriceInPurchaseCurrencyWithCurrencySymbol': 'US$ 8.05',
  'bestPriceInSurferCurrencyWithCurrencySymbol': 'US$ 8.05',
  'shippingToDestinationPriceInPurchaseCurrencyWithCurrencySymbol': 'US$ 0.00',
  'shippingToDestinationPriceInSurferCurrencyWithCurrencySymbol': 'US$ 0.00',
  'shippingDestinationNameInSurferLanguage': 'U.S.A.',
  'vendorCountryNameInSurferLanguage': 'U.S.A.',
  'vendorId': 51315977,
  'bestPriceInPurchaseCurrencyValueOnly': '8.05',
  'bestShippingToDestinationPriceInPurchaseCurrencyValueOnly': '0.00',
  'purchaseCurrencySymbol': 'US$',
  'refinementList': None,
  'internationalEdition': False,
  'bookCondition': 'Good',
  'bookDescription': 'Former library book; may include...',
  'freeShipping': True},
 'pricingInfoForBestAllConditions': None,
 'isbn': '9781250297662',
 'totalResults': 18,
 'containerId': 'pricingService-9781250297662',
 'refinementList': [{'name': 'collectibleSigned',
   'label': 'Signed',
   'count': 1,
   'url': 'attrs=sgnd&ds=5&isbn=9781250297662&sgnd=on&sortby=17',
   'soutaRequest': {'debug': None,
    'rollup': None,
    'deep': None,
    'facets': True,
    'bsaExpandedFacets': None,
    'strict': None,
    'shippingDestination': 'USA',
    'shippingMode': None,
    'currency': 'USD',
    'clientId': None,
    'sort': 'TOTAL_PRICE_ASC',
    'pageSize': 5,
    'pageDescriptor': None,
    'featureFlags': None,
    'query': {'domain': 'COM',
     'booleanQuery': None,
     'isbns': [{'isbn13': True, 'isbn10': False}],
     'contributor': None,
     'title': None,
     'publisher': None,
     'keyword': None,
     'minYearPublished': None,
     'maxYearPublished': None,
     'minPrice': None,
     'maxPrice': None,
     'createdWithin': None,
     'minCreatedTimestamp': None,
     'maxCreatedTimestamp': None,
     'includedLanguages': None,
     'includedProductTypes': None,
     'requiredAttributes': ['SIGNED'],
     'includedConditions': None,
     'requiredBsaCodes': None,
     'requiredFreeShippingCountries': None,
     'minSellerRating': None,
     'sellerProvState': None,
     'sellerCountry': None,
     'includedSellerRegions': None,
     'sellerCatalog': None,
     'requiredSellerGroups': None,
     'includedSellers': None,
     'excludedSellers': None,
     'includedListingIds': None,
     'excludeSellerPictures': None}}},
  {'name': 'collectibleJacket',
   'label': 'Dust Jacket',
   'count': 1,
   'url': 'attrs=dj&dj=on&ds=5&isbn=9781250297662&sortby=17',
   'soutaRequest': {'debug': None,
    'rollup': None,
    'deep': None,
    'facets': True,
    'bsaExpandedFacets': None,
    'strict': None,
    'shippingDestination': 'USA',
    'shippingMode': None,
    'currency': 'USD',
    'clientId': None,
    'sort': 'TOTAL_PRICE_ASC',
    'pageSize': 5,
    'pageDescriptor': None,
    'featureFlags': None,
    'query': {'domain': 'COM',
     'booleanQuery': None,
     'isbns': [{'isbn13': True, 'isbn10': False}],
     'contributor': None,
     'title': None,
     'publisher': None,
     'keyword': None,
     'minYearPublished': None,
     'maxYearPublished': None,
     'minPrice': None,
     'maxPrice': None,
     'createdWithin': None,
     'minCreatedTimestamp': None,
     'maxCreatedTimestamp': None,
     'includedLanguages': None,
     'includedProductTypes': None,
     'requiredAttributes': ['JACKET'],
     'includedConditions': None,
     'requiredBsaCodes': None,
     'requiredFreeShippingCountries': None,
     'minSellerRating': None,
     'sellerProvState': None,
     'sellerCountry': None,
     'includedSellerRegions': None,
     'sellerCatalog': None,
     'requiredSellerGroups': None,
     'includedSellers': None,
     'excludedSellers': None,
     'includedListingIds': None,
     'excludeSellerPictures': None}}},
  {'name': 'bindingHard',
   'label': 'Hardcover',
   'count': 18,
   'url': 'attrs=hc&bi=h&ds=5&isbn=9781250297662&sortby=17',
   'soutaRequest': {'debug': None,
    'rollup': None,
    'deep': None,
    'facets': True,
    'bsaExpandedFacets': None,
    'strict': None,
    'shippingDestination': 'USA',
    'shippingMode': None,
    'currency': 'USD',
    'clientId': None,
    'sort': 'TOTAL_PRICE_ASC',
    'pageSize': 5,
    'pageDescriptor': None,
    'featureFlags': None,
    'query': {'domain': 'COM',
     'booleanQuery': None,
     'isbns': [{'isbn13': True, 'isbn10': False}],
     'contributor': None,
     'title': None,
     'publisher': None,
     'keyword': None,
     'minYearPublished': None,
     'maxYearPublished': None,
     'minPrice': None,
     'maxPrice': None,
     'createdWithin': None,
     'minCreatedTimestamp': None,
     'maxCreatedTimestamp': None,
     'includedLanguages': None,
     'includedProductTypes': None,
     'requiredAttributes': ['HARDCOVER'],
     'includedConditions': None,
     'requiredBsaCodes': None,
     'requiredFreeShippingCountries': None,
     'minSellerRating': None,
     'sellerProvState': None,
     'sellerCountry': None,
     'includedSellerRegions': None,
     'sellerCatalog': None,
     'requiredSellerGroups': None,
     'includedSellers': None,
     'excludedSellers': None,
     'includedListingIds': None,
     'excludeSellerPictures': None}}},
  {'name': 'collectibleFirstEdition',
   'label': 'First Edition',
   'count': 1,
   'url': 'attrs=fe&ds=5&fe=on&isbn=9781250297662&sortby=17',
   'soutaRequest': {'debug': None,
    'rollup': None,
    'deep': None,
    'facets': True,
    'bsaExpandedFacets': None,
    'strict': None,
    'shippingDestination': 'USA',
    'shippingMode': None,
    'currency': 'USD',
    'clientId': None,
    'sort': 'TOTAL_PRICE_ASC',
    'pageSize': 5,
    'pageDescriptor': None,
    'featureFlags': None,
    'query': {'domain': 'COM',
     'booleanQuery': None,
     'isbns': [{'isbn13': True, 'isbn10': False}],
     'contributor': None,
     'title': None,
     'publisher': None,
     'keyword': None,
     'minYearPublished': None,
     'maxYearPublished': None,
     'minPrice': None,
     'maxPrice': None,
     'createdWithin': None,
     'minCreatedTimestamp': None,
     'maxCreatedTimestamp': None,
     'includedLanguages': None,
     'includedProductTypes': None,
     'requiredAttributes': ['FIRST_EDITION'],
     'includedConditions': None,
     'requiredBsaCodes': None,
     'requiredFreeShippingCountries': None,
     'minSellerRating': None,
     'sellerProvState': None,
     'sellerCountry': None,
     'includedSellerRegions': None,
     'sellerCatalog': None,
     'requiredSellerGroups': None,
     'includedSellers': None,
     'excludedSellers': None,
     'includedListingIds': None,
     'excludeSellerPictures': None}}}],
 'bibliographicDetail': None}

url = "https://www.abebooks.com/servlet/RecommendationsApi"

# Get book recommendations by ISBN
payload = {'pageId': 'plp',
           'itemIsbn13': 9781250297662}

resp = requests.get(url, params=payload)
print(resp.status_code, resp.reason)
resp.json()

200 OK

{'widgetResponses': [{'slotName': 'detail-1',
    'title': 'Customers who bought this item also bought',
    'algoName': 'abeBooksBlendedPurchaseSims',
    'ref': 'pd_b_p_1',
    'recommendationItems': [{'attributes': [],
        'thumbNailImgUrl': 'https://pictures.abebooks.com/isbn/9780765384201-us-300.jpg',
        'itemLink': '/products/isbn/9780765384201?cm_sp=rec-_-pd_b_p_1-_-plp&reftag=pd_b_p_1',
        'subTitle': None,
        'isbn13': '9780765384201',
        'title': 'Invisible Planets: Contemporary Chinese Science Fiction...',
        'author': 'Liu, Ken'},
    {'attributes': [],
        'thumbNailImgUrl': 'https://pictures.abebooks.com/isbn/9781250306029-us-300.jpg',
        'itemLink': '/products/isbn/9781250306029?cm_sp=rec-_-pd_b_p_1-_-plp&reftag=pd_b_p_1',
        'subTitle': None,
        'isbn13': '9781250306029',
        'title': 'The Redemption of Time: A Three-Body Problem Novel...',
        'author': 'Baoshu'},
    {'attributes': [],
        'thumbNailImgUrl': 'https://pictures.abebooks.com/isbn/9780765389312-us-300.jpg',
        'itemLink': '/products/isbn/9780765389312?cm_sp=rec-_-pd_b_p_1-_-plp&reftag=pd_b_p_1',
        'subTitle': None,
        'isbn13': '9780765389312',
        'title': 'Waste Tide',
        'author': 'Qiufan, Chen'},
    {'attributes': [],
        'thumbNailImgUrl': 'https://pictures.abebooks.com/isbn/9780765384195-us-300.jpg',
        'itemLink': '/products/isbn/9780765384195?cm_sp=rec-_-pd_b_p_1-_-plp&reftag=pd_b_p_1',
        'subTitle': None,
        'isbn13': '9780765384195',
        'title': 'Invisible Planets: Contemporary Chinese Science Fiction...',
        'author': 'Liu, Ken'},
    {'attributes': [],
        'thumbNailImgUrl': 'https://pictures.abebooks.com/isbn/9781784978518-us-300.jpg',
        'itemLink': '/products/isbn/9781784978518?cm_sp=rec-_-pd_b_p_1-_-plp&reftag=pd_b_p_1',
        'subTitle': None,
        'isbn13': '9781784978518',
        'title': 'The Wandering Earth',
        'author': 'Liu, Cixin'}]},
    {'slotName': 'ext-search-detail-1',
    'title': None,
    'algoName': 'heroWidgetIsbnSims',
    'ref': 'pd_hw_i_1',
    'recommendationItems': [{'attributes': [],
        'thumbNailImgUrl': 'https://pictures.abebooks.com/isbn/9780804172448-us-300.jpg',
        'itemLink': '/products/isbn/9780804172448?cm_sp=rec-_-pd_hw_i_1-_-plp&reftag=pd_hw_i_1',
        'subTitle': 'Best Selling',
        'isbn13': '9780804172448',
        'title': 'Station Eleven',
        'author': 'Mandel, Emily St. John'},
    {'attributes': [],
        'thumbNailImgUrl': 'https://pictures.abebooks.com/isbn/9781786073495-us-300.jpg',
        'itemLink': '/products/isbn/9781786073495?cm_sp=rec-_-pd_hw_i_1-_-plp&reftag=pd_hw_i_1',
        'subTitle': 'Top Rated',
        'isbn13': '9781786073495',
        'title': 'Zuleikha',
        'author': 'Yakhina, Guzel'}]}]}

An Object-Oriented Module

I created a small Python module abebooks.py to encapsulate the requests. The full code is below:

import requests


class AbeBooks:

    def __get_price(self, payload):
        url = "https://www.abebooks.com/servlet/DWRestService/pricingservice"
        resp = requests.post(url, data=payload)
        resp.raise_for_status()
        return resp.json()

    def __get_recomendations(self, payload):
        url = "https://www.abebooks.com/servlet/RecommendationsApi"
        resp = requests.get(url, params=payload)
        resp.raise_for_status()
        return resp.json()

    def getPriceByISBN(self, isbn):
        """
        Parameters
        ----------
        isbn (int) - a book's ISBN code
        """
        payload = {'action': 'getPricingDataByISBN',
                   'isbn': isbn,
                   'container': 'pricingService-{}'.format(isbn)}
        return self.__get_price(payload)

    def getPriceByAuthorTitle(self, author, title):
        """
        Parameters
        ----------
        author (str) - book author
        title (str) - book title
        """
        payload = {'action': 'getPricingDataForAuthorTitleStandardAddToBasket',
                   'an': author,
                   'tn': title,
                   'container': 'oe-search-all'}
        return self.__get_price(payload)

    def getPriceByAuthorTitleBinding(self, author, title, binding):
        """
        Parameters
        ----------
        author (str) - book author
        title (str) - book title
        binding(str) - one of 'hard', or 'soft'
        """
        if binding == "hard":
            container = "priced-from-hard"
        elif binding == "soft":
            container = "priced-from-soft"
        else:
            raise ValueError(
                    'Invalid parameter. Binding must be "hard" or "soft"')
        payload = {'action': 'getPricingDataForAuthorTitleBindingRefinements',
                   'an': author,
                   'tn': title,
                   'container': container}
        return self.__get_price(payload)

    def getRecommendationsByISBN(self, isbn):
        """
        Parameters
        ----------
        isbn (int) - a book's ISBN code
        """
        payload = {'pageId': 'plp',
                   'itemIsbn13': isbn}
        return self.__get_recomendations(payload)

from abebooks import AbeBooks

ab = AbeBooks()
results = ab.getPriceByISBN(9780062941503)
if results['success']:
    best_new = results['pricingInfoForBestNew']
    best_used = results['pricingInfoForBestUsed']

# Best New Price
print(best_new['bestPriceInPurchaseCurrencyWithCurrencySymbol'])

US$ 21.49

# Best Used Price
print(best_used['bestPriceInPurchaseCurrencyWithCurrencySymbol'])

US$ 24.42

Reticulate

install.packages("reticulate")

R Code

library(reticulate)

Python Code

import numpy as np
import matplotlib.pyplot as plt

x = np.array([1, 2, 3, 4, 5])
y = np.exp2(x)

plt.plot(x, y)
plt.show()

R Code

np <- import("numpy", "np")
x <- np$array(c(1, 2, 3, 4, 5))
y <- np$exp2(x)

R Code

py_run_string("
import numpy as np
x = np.array([1,2,3,4,5])
y = np.exp2(x)
")

py_run_file("python_script.py")

R Code

# Plotting the same arrays in R! So simple!
plot(py$x, py$y)

R Code

data(mtcars)

Python Code

# Python Code
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler

# Access mtcars from R and do some Python-specific analysis
mtcars_py = r.mtcars

numeric_cols = ['mpg', 'disp', 'hp', 'drat', 'wt', 'qsec']
1scaler = StandardScaler()
mtcars_py[numeric_cols] = scaler.fit_transform(mtcars_py[numeric_cols])
mtcars_py.head()

                        mpg  cyl      disp        hp  ...   vs   am  gear  carb
Mazda RX4          0.153299  6.0 -0.579750 -0.543655  ...  0.0  1.0   4.0   4.0
Mazda RX4 Wag      0.153299  6.0 -0.579750 -0.543655  ...  0.0  1.0   4.0   4.0
Datsun 710         0.456737  4.0 -1.006026 -0.795570  ...  1.0  1.0   4.0   1.0
Hornet 4 Drive     0.220730  6.0  0.223615 -0.543655  ...  1.0  0.0   3.0   1.0
Hornet Sportabout -0.234427  8.0  1.059772  0.419550  ...  0.0  0.0   3.0   2.0

[5 rows x 11 columns]

Python Code

# Convert the Pandas dataframe to a dictionary
mtcars_dict = mtcars_py.to_dict(orient='list')

R Code

library(ggplot2)

# Convert the dictionary to an R data frame
standardized_data <- data.frame(py$mtcars_dict)

ggplot(standardized_data, aes(x = wt, y = mpg, color = factor(cyl))) +
  geom_point() +
  theme_minimal() +
  labs(title = "Standardized Weight vs MPG by Cylinders",
       x = "Standardized Weight",
       y = "Standardized MPG",
       color = "Cylinders")

Using Apache Arrow for Large Data Interchange

For sharing large datasets, you might want to consider Apache Arrow. Arrow provides a language-agnostic columnar memory format, enabling fast data exchange between R, Python, and other systems.

R Code

1suppressPackageStartupMessages(library(arrow))

2mtcars_arrow <- arrow_table(mtcars)

Python Code

import pyarrow as pa

mtcars_py = r.mtcars_arrow
# An aggregation example in pyarrow
mtcars_py.group_by('cyl').aggregate([('mpg', 'mean')]).to_pandas()

   cyl   mpg_mean
0  6.0  19.742857
1  4.0  26.663636
2  8.0  15.100000

Python Code

# Convert a Pandas DataFrame to an Arrow table
df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})
table = pa.Table.from_pandas(df)

R Code

# Access the arrow table from R
df <- py$table
1as.data.frame(df)

  a b
1 1 4
2 2 5
3 3 6

Wrapping up

Reticulate makes interfacing Python and R code nearly seamless, especially when working in a Quarto notebook, and with the addition of Apache Arrow, we can efficiently share large datasets between the two languages.

However, while Reticulate works great if you are working in an R-based environment, it is also common to start out in a Python environment and have need of an R library (e.g., a Bioconductor package or plotting function). In such cases, we can use rpy2, a Python package that provides an interface to call R code from Python. Additionally, rpy2-arrow provides an extension for using Apache Arrow with rpy2. It would probably make this blog post too long to cover rpy2 in detail, but this blog did an excellent job of explaining how to use it: Data transfer between Python and R with rpy2 and Apache Arrow.

I am excited to try out these workflows more extensively, and leverage some of the strengths of both languages via some Franken-pipelines! What are your favorite use cases for combining Python and R?

References and Further Reading

• Apache Arrow R Cookbook
• Data transfer between Python and R with rpy2 and Apache Arrow.
• Calling R from Python with rpy2

Getting Target Data from Pharos

First, we connect to the Pharos database with the help of SQLAlchemy and read the necessary tables into Pandas DataFrames.

import pandas as pd
from sqlalchemy import create_engine

connection_string = "mysql+mysqlconnector://tcrd@tcrd.kmc.io/tcrd540"
engine = create_engine(connection_string)

# Get targets in Tclin and Tchem
query_target = """
  SELECT id, name, tdl, fam, famext
  FROM target
  WHERE tdl IN ('Tclin','Tchem')
"""
target = pd.read_sql(query_target, con=engine)

# Get Uniprot ids
query_protein = "SELECT id, uniprot, family FROM protein"
protein = pd.read_sql(query_protein, con=engine)

# Get compound activities
query_cmpd_activity = """
  SELECT id, target_id, act_value
  FROM cmpd_activity
"""

cmpd_activity = pd.read_sql(query_cmpd_activity, con=engine)

Here are the datasets that we obtained:

target.head()

protein.head()

cmpd_activity.head()

Filtering targets with more than 15 active compounds

actives_count = (
    cmpd_activity
    .groupby('target_id', as_index=False)
    .agg(num_actives=('id','count'))
)

# Merge with the target table
target = target.merge(actives_count,
                      left_on='id',
                      right_on='target_id',
                      how='left')

# Keep only targets with >= 15 active compounds
target = target[target['num_actives'] >= 15]

At this point, we have a table of Tclin/Tchem targets that each have at least 15 active compounds recorded in Pharos.

Read SIFTS Mappings

The SIFTS PDB chain to UniProt mappings were downloaded from their ftp site.

import requests
import tempfile
from typing import Dict, List

sifts_url = "https://ftp.ebi.ac.uk/pub/databases/msd/sifts/flatfiles/csv/pdb_chain_uniprot.csv.gz"

dtypes: Dict[str, str] = {
    'PDB': str,
    'CHAIN': str,
    'SP_PRIMARY': str,
    'RES_BEG': int,
    'RES_END': int,
    'PDB_BEG': str,  # Keep as string since it may contain insertion codes
    'PDB_END': str,  # Keep as string since it may contain insertion codes
    'SP_BEG': int,
    'SP_END': int
}

with tempfile.TemporaryDirectory() as tmpdirname:
    response = requests.get(sifts_url)
    gz_file_path = f"{tmpdirname}/pdb_chain_uniprot.csv.gz"
    with open(gz_file_path, "wb") as gz_file:
        gz_file.write(response.content)  # Raw compressed bytes
    # Read the gzipped CSV with pandas
    sifts_data = pd.read_csv(
        gz_file_path,
        compression="gzip",
        comment="#",
        dtype=dtypes
    )

# Rename the columns
sifts_data.rename(
    columns={
        "PDB": "pdb",
        "CHAIN": "chain_id",
        "SP_PRIMARY": "uniprot",
        "RES_BEG": "res_beg",
        "RES_END": "res_end",
        "PDB_BEG": "pdb_res_beg",
        "PDB_END": "pdb_res_end",
        "SP_BEG": "uniprot_res_beg",
        "SP_END": "uniprot_res_end",
    },
    inplace=True
)

sifts_data.head(5)

For this simple analysis, we only care whether a target has at least one PDB structure, so we can use the pdb (PDB) and uniprot (SP_PRIMARY) columns, but if we wanted to do a structure-based analysis and download the associated PDBs, we could use the residue-level mappings to identify the exact ranges of our PDB that we need to extract, as well as identify any missing residues that may need to be filled.

Now, we can group the SIFTS data by UniProt ID and get a list of unique PDB IDs for each target:

# Group by uniprot_id and get unique PDB IDs
uniprot_to_pdb = (
    sifts_data
    .groupby("uniprot", as_index=False)
    .agg(pdb_ids=("pdb", lambda x: ';'.join(x.unique())))
)
uniprot_to_pdb.head(5)

Merging Target Data with SIFTS

Since our target table has a corresponding protein table (linked by protein.id == target.id in TCRD) and each protein row has a UniProt ID, we can merge the two, and then merge with SIFTS on the UniProt accession:

# Filter the protein table to only those in our target set
protein_subset = protein[protein.id.isin(target.id)]

# Merge target and protein by TCRD 'id'
merged_df = target.merge(protein_subset, left_on='id', right_on='id', how='left')

# Merge SIFTS data
merged_df = merged_df.merge(uniprot_to_pdb, left_on='uniprot', right_on='uniprot', how='left')

# Rename columns for clarity
merged_df.rename(columns={'fam': 'target_family',
                   'tdl': 'target_dev_level'}, inplace=True)

merged_df['target_family'] = merged_df['target_family'].fillna('None', inplace=False)

# Mark whether a target has at least one known PDB structure
merged_df['has_pdb'] = ~merged_df['pdb_ids'].isna()

merged_df.head()

Visualizing the Data

Here, we visualize the count of targets with and without PDB structures, as well as the counts by target family.

import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from typing import Tuple

# Set global style

def plot_overall_pdb_counts(df: pd.DataFrame) -> Tuple[plt.Figure, plt.Axes]:
    """
    Plot the number of targets with and without PDB structures.
    """
    total = len(df)
    counts = df['has_pdb'].value_counts()
    percentages = (counts / total * 100).round(1)

    fig, ax = plt.subplots(figsize=(6, 4))
    width = 0.6
    x = np.arange(2)
    colors = ['#66c2a5', '#fc8d62']
    bars = ax.bar(x, counts.values, width, color=colors, zorder=2)

    for bar in bars:
        height = bar.get_height()
        pct = (height/total * 100).round(1)
        ax.text(bar.get_x() + bar.get_width()/2, height, f'{int(height)}\n({pct}%)', ha='center', va='bottom', size=9)

    ax.set_xticks(x)
    ax.set_xticklabels(['Has structure', 'No structure'])
    ax.set_ylabel('Number of targets')
    ax.yaxis.grid(True, linestyle='-', alpha=0.2, zorder=1)
    ax.xaxis.grid(False)
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    plt.tight_layout()

    return fig, ax

def plot_pdb_counts_by_family(df: pd.DataFrame) -> Tuple[plt.Figure, plt.Axes]:
    """
    Plot the number of targets with and without PDB structures, grouped by target family.
    """
    family_counts = df.groupby(['target_family', 'has_pdb']).size().unstack(fill_value=0)
    family_totals = family_counts.sum(axis=1)
    family_percentages = (family_counts.div(family_totals, axis=0) * 100).round(1)
    family_counts = family_counts.loc[family_counts.sum(axis=1).sort_values(ascending=False).index]

    fig, ax = plt.subplots(figsize=(8, 4))
    width = 0.35

    x = np.arange(len(family_counts.index))
    bars1 = ax.bar(x - width/2, family_counts[True], width, label='Has structure', color='#66c2a5', zorder=2)
    bars2 = ax.bar(x + width/2, family_counts[False], width, label='No structure', color='#fc8d62', zorder=2)

    def add_value_labels(bars):
        for bar in bars:
            height = bar.get_height()
            if height > 0:
                    ax.text(bar.get_x() + bar.get_width()/2, height, f'{int(height)}', ha='center', va='bottom', fontsize=8)

    add_value_labels(bars1)
    add_value_labels(bars2)

    ax.set_xticks(x)
    ax.set_xticklabels(family_counts.index, rotation=45, ha='right')
    ax.set_ylabel('Number of targets')
    ax.yaxis.grid(True, linestyle='-', alpha=0.2, zorder=1)
    ax.xaxis.grid(False)
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.legend(loc='upper right', frameon=True, facecolor='white', framealpha=0.9, edgecolor='none')
    plt.tight_layout()

    return fig, ax

fig, ax = plot_overall_pdb_counts(merged_df)
plt.show()

fix, ax = plot_pdb_counts_by_family(merged_df)
plt.show()

Conclusion

This is a basic example of how we can combine data from Pharos and SIFTS to identify targets with known active compounds and available PDB structures. There are many additional filters and considerations we could apply depending on our specific research questions and goals.

For example, bringing in additional data from the PDB to filter the structures based on resolution, completeness, or other structural features could help refine our target selection further.