Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
65 commits
Select commit Hold shift + click to select a range
7e68c2d
add notebook
cornhundred Jul 23, 2025
ccc44df
working in notebook
cornhundred Jul 24, 2025
5735714
working on notebook
cornhundred Jul 24, 2025
04623f2
Merge branch 'main' into DEGA-306-sST-update
cornhundred Jul 24, 2025
e5cb8c3
working through pre-processing steps
cornhundred Jul 24, 2025
ec2395b
working on landscape viz
cornhundred Jul 24, 2025
9f154e4
removed hardwiring for defining image name
cornhundred Jul 24, 2025
12230b0
updated notebook
cornhundred Jul 25, 2025
accf25c
reduced notebook size
cornhundred Jul 25, 2025
61cfe54
Enable IST preprocessing (#177)
cornhundred Jul 25, 2025
b746379
renamed cell_metadata function to spot position
cornhundred Jul 25, 2025
2daea86
Use simple image layer for H&E
cornhundred Jul 27, 2025
88bd913
Generate IST cell metadata before gene parquet step
cornhundred Jul 28, 2025
c653992
Fix custom meta cell loading
cornhundred Jul 28, 2025
228967d
Fix gene parquet saving
cornhundred Jul 28, 2025
05fdb8b
Avoid in-place replace on sparse data
cornhundred Jul 28, 2025
d07be67
fix sparse to_parquet
cornhundred Jul 28, 2025
7e03e71
fix gene parquet saving
cornhundred Jul 28, 2025
b9ca329
fix parquet gene filtering with duplicate names
cornhundred Jul 28, 2025
350f3f3
Fix gene parquet export with duplicate columns
cornhundred Jul 28, 2025
d93e24d
addressing duplicate gene names before making cbg files
cornhundred Jul 28, 2025
8fc203c
Use simple image layer for H&E (#179)
cornhundred Jul 28, 2025
0bb7e3b
debugging
cornhundred Jul 28, 2025
affc296
fixed notebook
cornhundred Jul 28, 2025
93c71ff
feat: skip preprocessing steps when outputs exist (#181)
cornhundred Jul 28, 2025
4d709cd
working on reading cell boundaries
cornhundred Jul 28, 2025
a7afddb
working on paths
cornhundred Jul 28, 2025
cb641ff
debugging paths
cornhundred Jul 28, 2025
1ba14a1
debugging paths
cornhundred Jul 28, 2025
ac8ca24
debugging paths
cornhundred Jul 28, 2025
2338c42
paths
cornhundred Jul 29, 2025
ec7d767
debugging
cornhundred Jul 29, 2025
bd0f619
meta_cell working
cornhundred Jul 29, 2025
1e7a5a0
working on cell boundaries
cornhundred Jul 29, 2025
03cee15
fix IST boundary tiling and transcripts (#183)
cornhundred Jul 29, 2025
0a4bf40
resolved conflicts
cornhundred Jul 29, 2025
ecd130f
organizing
cornhundred Jul 29, 2025
86805f4
cleaning
cornhundred Jul 29, 2025
78668a7
fixed mtx barcode parsing error with IST when loading cbg
cornhundred Jul 29, 2025
23df4ec
fixing sbg/cbg mixup
cornhundred Jul 29, 2025
51c7165
spot index matching
cornhundred Jul 29, 2025
3692871
following previous approach for jitter
cornhundred Jul 29, 2025
2723a30
working on jitter
cornhundred Jul 29, 2025
44f3484
nm to um
cornhundred Jul 29, 2025
64f2972
dubugging
cornhundred Jul 29, 2025
19a6d6e
fixing file path
cornhundred Jul 29, 2025
c5c9e54
transcripts tiles are starting to work
cornhundred Jul 29, 2025
bae692f
using vectorized approach to pseudo trx
cornhundred Jul 30, 2025
bc5b446
looking up gene index for trx parquets
cornhundred Jul 30, 2025
5d0064c
experimenting with using Polars to speed up pre-processing
cornhundred Jul 30, 2025
9bc82f9
testing pandas/polars hybrid
cornhundred Jul 30, 2025
b882128
IST viz starting to work
cornhundred Jul 30, 2025
ea89117
working on max_tiles_to_view argument
cornhundred Jul 30, 2025
44f3d2c
front-end testing
cornhundred Jul 31, 2025
adf1a89
fix cell meta location and orientation
huanlity Jul 31, 2025
9fec987
ruff format
huanlity Jul 31, 2025
b56a1a1
fix polygon index and cbg index
huanlity Aug 1, 2025
2964659
reverting front-end changes
cornhundred Aug 1, 2025
4bff445
cleaning and reverting
cornhundred Aug 1, 2025
8b21d84
adding console logs to track down slowdown
cornhundred Aug 1, 2025
dd294d5
updated notebooks
cornhundred Aug 1, 2025
e364510
Limit gene bar charts to top 1000 (#193)
cornhundred Aug 4, 2025
00b7823
reprocess and validate
huanlity Aug 4, 2025
10fbb2a
merg main
huanlity Aug 4, 2025
112a221
add notebook
huanlity Aug 6, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions js/deck-gl/core/calc_viewport.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ export const calc_viewport = async (
viz_state
) => {
const { tile_size } = viz_state.img.landscape_parameters;
const max_tiles_to_view = 50;
const zoomFactor = Math.pow(2, zoom);
const [targetX, targetY] = target;
const halfWidthZoomed = width / (2 * zoomFactor);
Expand All @@ -35,7 +34,7 @@ export const calc_viewport = async (
tile_size
);

if (tiles_in_view.length < max_tiles_to_view) {
if (tiles_in_view.length < viz_state.max_tiles_to_view) {
viz_state.obs_store.deck_check.set({
...viz_state.obs_store.deck_check.get(),
trx_data: false,
Expand Down Expand Up @@ -82,7 +81,8 @@ export const calc_viewport = async (
return acc;
}, [])
.filter((item) => item.value > 0)
.sort((a, b) => b.value - a.value);
.sort((a, b) => b.value - a.value)
.slice(0, 100);

viz_state.obs_store.new_gene_bar_data.set(new_bar_data);

Expand Down Expand Up @@ -115,7 +115,9 @@ export const calc_viewport = async (
if (viz_state.close_up) {
viz_state.close_up = false;

viz_state.obs_store.new_gene_bar_data.set(viz_state.genes.gene_counts);
viz_state.obs_store.new_gene_bar_data.set(
viz_state.genes.top_gene_counts
);

viz_state.obs_store.new_cell_bar_data.set(viz_state.cats.cluster_counts);

Expand Down
14 changes: 13 additions & 1 deletion js/deck-gl/layers/image_layers.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import {
create_render_tile_sublayers,
} from '../utils/tiles';

import { make_simple_image_layer } from './simple_image_layer';

const make_image_layer = (viz_state, info) => {
const { max_pyramid_zoom } = viz_state.img.landscape_parameters;

Expand Down Expand Up @@ -37,7 +39,17 @@ const make_image_layer = (viz_state, info) => {
};

export const make_image_layers = async (viz_state) => {
const image_layers = viz_state.img.image_info.map((info) =>
const { image_info } = viz_state.img;

if (
image_info.length === 1 &&
(image_info[0].name === 'h_and_e' || image_info[0].name === 'h&e')
) {
const layer = await make_simple_image_layer(viz_state, image_info[0]);
return [layer];
}

const image_layers = image_info.map((info) =>
make_image_layer(viz_state, info)
);
return image_layers;
Expand Down
14 changes: 13 additions & 1 deletion js/global_variables/cell_exp_array.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,22 @@
vector_name_integer,
aws
) => {

console.log('------------------------------------------------------')

Check warning on line 20 in js/global_variables/cell_exp_array.js

View workflow job for this annotation

GitHub Actions / quality

Unexpected console statement
console.log('in update_cell_exp_array function')

Check warning on line 21 in js/global_variables/cell_exp_array.js

View workflow job for this annotation

GitHub Actions / quality

Unexpected console statement

let file_path;
if (version === 'default') {
file_path = `${base_url}/cbg/${inst_gene}.parquet`;
} else {
file_path = `${base_url}/cbg_${version}/${inst_gene}.parquet`;
}

//var file_path = base_url + '/cbg/' + inst_gene + '.parquet'
console.log('Fetching expression table from:', file_path);

Check warning on line 30 in js/global_variables/cell_exp_array.js

View workflow job for this annotation

GitHub Actions / quality

Unexpected console statement
const exp_table = await get_arrow_table(file_path, options.fetch, aws);
console.log('get cell_names')

Check warning on line 32 in js/global_variables/cell_exp_array.js

View workflow job for this annotation

GitHub Actions / quality

Unexpected console statement
const cell_names = exp_table.getChild('__index_level_0__').toArray();
console.log('get cell_exp')

Check warning on line 34 in js/global_variables/cell_exp_array.js

View workflow job for this annotation

GitHub Actions / quality

Unexpected console statement
const cell_exp = exp_table.getChild(inst_gene).toArray();

const new_exp_array = new Array(cats.cell_names_array.length).fill(0);
Expand All @@ -36,6 +42,7 @@
const missingCellNames1 = new Set(); // For cell_name_to_index_map
const missingCellNames2 = new Set(); // For nameMapping_inv

console.log('Updating cell_exp_array for gene:', inst_gene);

Check warning on line 45 in js/global_variables/cell_exp_array.js

View workflow job for this annotation

GitHub Actions / quality

Unexpected console statement
cell_names.forEach((name, i) => {
name = String(name);
const exp_value = Number(cell_exp[i]);
Expand All @@ -57,6 +64,8 @@
}
});

console.log('done updating cell_exp_array for gene:', inst_gene);

Check warning on line 67 in js/global_variables/cell_exp_array.js

View workflow job for this annotation

GitHub Actions / quality

Unexpected console statement

// Log missing names (if any) after processing all cells
if (missingCellNames1.size > 0) {
// console.log(`Cell names not found in cell_name_to_index_map (${missingCellNames1.size} unique names):`,
Expand All @@ -70,4 +79,7 @@
}

cats.cell_exp_array = new_exp_array;

console.log('end of update_cell_exp_array function')

Check warning on line 83 in js/global_variables/cell_exp_array.js

View workflow job for this annotation

GitHub Actions / quality

Unexpected console statement
console.log('------------------------------------------------------')

Check warning on line 84 in js/global_variables/cell_exp_array.js

View workflow job for this annotation

GitHub Actions / quality

Unexpected console statement
};
3 changes: 2 additions & 1 deletion js/global_variables/meta_gene.js
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ export const set_meta_gene = async (
});

genes.gene_counts.sort((a, b) => b.value - a.value);
genes.top_gene_counts = genes.gene_counts.slice(0, 100);

// Create the reverse mapping: integer index to gene name
const g_nameMapping_inv = gene_names.reduce((acc, name, idx) => {
Expand All @@ -50,5 +51,5 @@ export const set_meta_gene = async (
// Save the mapping as cats.nameMapping_inv
genes.g_nameMapping_inv = g_nameMapping_inv;

genes.gene_names = genes.gene_counts.map((gene) => gene.name);
genes.gene_names = gene_names;
};
15 changes: 13 additions & 2 deletions js/read_parquet/get_arrow_table.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,22 @@

export const get_arrow_table = async (url, fetch_options, aws) => {
try {
const response =
aws !== null ? await aws.fetch(url) : await fetch(url, fetch_options);

console.log('fetch')

Check warning on line 8 in js/read_parquet/get_arrow_table.js

View workflow job for this annotation

GitHub Actions / quality

Unexpected console statement

console.log('aws:', aws);

// const response =
// aws !== null ? await aws.fetch(url) : await fetch(url, fetch_options);


const response = await fetch(url, fetch_options);

console.log('arrayBuffer')
const arrayBuffer = await response.arrayBuffer();
console.log('arrowTable');
const arrowTable = arrayBufferToArrowTable(arrayBuffer);
console.log('return arrowTable')
return arrowTable;
} catch (error) {
handleAsyncError(error, {
Expand Down
13 changes: 11 additions & 2 deletions js/ui/bar_plot.js
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ export const bar_callback_gene = async (
_layers_obj,
_viz_state
) => {

console.log('bar_callback_gene', d.name);

// ensure that trx button, slider, and bars are active
_viz_state.buttons.buttons.trx.style('color', 'blue');

Expand All @@ -81,17 +84,22 @@ export const bar_callback_gene = async (
_viz_state.cats.svg_bar_cluster.selectAll('rect').style('opacity', 0.2);
}

console.log('update_cat', new_cat);
update_cat(_viz_state.cats, new_cat);

console.log('deck_check set for trx_layer and cell_layer');
_viz_state.obs_store.deck_check.set({
..._viz_state.obs_store.deck_check.get(),
cell_layer: false,
trx_layer: false,
});

console.log('update_selected_genes', [inst_gene]);
update_selected_genes(_viz_state.genes, [inst_gene], _viz_state.obs_store);
// testing setting selected_cats to array with the selected gene for
// observable updates
console.log('update_selected_cats', [inst_gene]);
update_selected_cats(_viz_state.cats, [inst_gene], _viz_state.obs_store);

console.log('****** update_cell_exp_array for gene:', inst_gene);
await update_cell_exp_array(
_viz_state.cats,
_viz_state.genes,
Expand All @@ -101,6 +109,7 @@ export const bar_callback_gene = async (
_viz_state.vector_name_integer,
_viz_state.aws
);
console.log('****** done updating cell_exp_array for gene:', inst_gene);
};

export const bar_callback_nbhd = (
Expand Down
2 changes: 1 addition & 1 deletion js/ui/ui_containers.js
Original file line number Diff line number Diff line change
Expand Up @@ -587,7 +587,7 @@ export const make_ist_ui_container = (
viz_state.containers.bar_gene,
bar_callback_gene,
viz_state.genes.svg_bar_gene,
viz_state.genes.gene_counts,
viz_state.genes.top_gene_counts,
viz_state.genes.color_dict_gene,
deck_ist,
layers_obj,
Expand Down
15 changes: 11 additions & 4 deletions js/viz/landscape_ist.js
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ export const landscape_ist = async (
landscape_state = 'spatial',
segmentation = 'default',
creds = {},
max_tiles_to_view = 50,
view_change_custom_callback = null
) => {
if (width === 0) {
Expand All @@ -93,7 +94,10 @@ export const landscape_ist = async (

viz_state.obs_store = create_obs_store();

viz_state.max_tiles_to_view = max_tiles_to_view;
const update_viz_image_layers = () => {

console.log('update_viz_image_layers called');
if (!get_img_layer_visible()) {
return;
}
Expand Down Expand Up @@ -271,13 +275,13 @@ export const landscape_ist = async (

set_options(token);

// move this to landscape_parameters
const imgage_name_for_dim = 'dapi';

await set_landscape_parameters(viz_state.img, base_url, viz_state.aws);

const tmp_image_info = viz_state.img.landscape_parameters.image_info;

// set image_name_for_dim using the first image info name
const image_name_for_dim = tmp_image_info[0].name;

viz_state.vector_name_integer =
viz_state.img.landscape_parameters.use_int_index;

Expand All @@ -297,7 +301,7 @@ export const landscape_ist = async (
root.style.height = `${height}px`;
root.style.border = '1px solid #d3d3d3';

await set_dimensions(viz_state, base_url, imgage_name_for_dim);
await set_dimensions(viz_state, base_url, image_name_for_dim);

await set_meta_gene(
viz_state.genes,
Expand Down Expand Up @@ -429,13 +433,15 @@ export const landscape_ist = async (
set_nbhd_layer_onclick(deck_ist, layers_obj, viz_state);

viz_state.obs_store.deck_ready.subscribe((ready) => {
console.log('deck_ready.subscribe', ready);
if (ready) {
const list = get_layers_list(viz_state.layers_obj, viz_state.close_up);
deck_ist.setProps({ layers: list });
}
});

viz_state.obs_store.selected_cats.subscribe((selected_cats) => {
console.log('selected_cats.subscribe', selected_cats);
const selected_cats_name = selected_cats.join('-');

layers_obj.cell_layer = layers_obj.cell_layer.clone({
Expand All @@ -454,6 +460,7 @@ export const landscape_ist = async (
});

viz_state.obs_store.selected_genes.subscribe((selected_genes) => {
console.log('selected_genes.subscribe', selected_genes);
const selected_genes_name = selected_genes.join('-');
layers_obj.trx_layer = layers_obj.trx_layer.clone({
id: `trx-layer-${selected_genes_name}`,
Expand Down
5 changes: 3 additions & 2 deletions js/widget.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ const render_landscape_ist = async ({ model, el }) => {
const dataset_name = model.get('dataset_name');
const width = model.get('width');
const height = model.get('height');

const nbhd = model.get('nbhd_geojson');
const max_tiles_to_view = model.get('max_tiles_to_view');

let meta_cell_data = { result: {}, attr: [] };
let meta_cluster_data = { result: {}, attr: [] };
Expand Down Expand Up @@ -64,7 +64,8 @@ const render_landscape_ist = async ({ model, el }) => {
nbhd,
landscape_state,
segmentation,
creds
creds,
max_tiles_to_view
);
};

Expand Down
Loading
Loading