ImgLib2 and BigDataViewer projects

In [1]:

%mavenRepo scijava.public https://maven.scijava.org/content/groups/public
%maven org.scijava:scijava-common:2.97.0
%maven net.imglib2:imglib2:6.2.0
%maven org.janelia.saalfeldlab:n5:3.1.2
%maven org.janelia.saalfeldlab:n5-imglib2:7.0.0
%maven org.janelia.saalfeldlab:n5-universe:1.3.1

In [2]:

import java.nio.file.*;
import java.util.stream.*;
import java.util.concurrent.*;

import com.google.gson.*;

import net.imglib2.*;
import net.imglib2.img.array.*;
import net.imglib2.type.numeric.real.*;
import net.imglib2.view.*;
import net.imglib2.util.*;

import org.janelia.saalfeldlab.n5.*;
import org.janelia.saalfeldlab.n5.imglib2.*;
import org.janelia.saalfeldlab.n5.universe.*;

In [3]:

public static void pathInfo(Path p) {
    try {
        System.out.println(String.format("%s is %d bytes", p, Files.size(p))); 
    } catch(IOException e ){}
}

public static void printBlocks(String path) throws IOException {

    try (Stream<Path> stream = Files.walk(Paths.get(path))) {
        stream.filter(Files::isRegularFile)
            .filter( p -> p.getFileName().toString().matches("[0-9]"))
                .forEach( x -> { pathInfo(x); });
    }
}

In [4]:

// N5Factory can make N5Readers and N5Writers
var factory = new N5Factory();

// trying to open a reader for a container that does not yet exist will throw an error 
// var n5Reader = factory.openReader("my-container.n5");

// creating a writer creates a container at the given location
// if it does not already exist
var n5Writer = factory.openWriter("my-container.n5");

// now we can make a reader
var n5Reader = factory.openReader("my-container.n5");

// test if the container exists
n5Reader.exists(""); // true

// "" and "/" both refer to the root of the container
n5Reader.exists("/"); // true

In [5]:

factory.openWriter("my-container.h5").getClass();   // HDF5 Format N5Writer
factory.openWriter("my-container.n5").getClass();   // N5 Format   N5Writer
factory.openWriter("my-container.zarr").getClass(); // Zarr Format N5Writer

In [6]:

n5Writer.createGroup("foo");
n5Writer.createGroup("foo/bar");
n5Writer.createGroup("lorum/ipsum/dolor/sit/amet");

n5Writer.exists("lorum/ipsum");      // true
n5Writer.exists("not/a/real/group"); // false

In [7]:

n5Writer.list("");     // [lorum, foo]
n5Writer.list("foo");  // [bar]

In [8]:

Arrays.toString(n5Writer.deepList(""));

[lorum, lorum/ipsum, lorum/ipsum/dolor, lorum/ipsum/dolor/sit, lorum/ipsum/dolor/sit/amet, foo, foo/bar]

In [9]:

public static RandomAccessibleInterval<FloatType> demoImage(long... size) {

    final RandomAccessibleInterval<FloatType> img = ArrayImgs.floats(size);
    float f = 0f;
    Cursor<FloatType> c = Views.flatIterable(img).cursor();
    while (c.hasNext())
        c.next().set(f++);

    return img;
}

In [10]:

// the parameters
var img = demoImage(64,64); // the image to write- size 64 x 64
var groupPath = "data"; 
var blockSize = new int[]{32,32};
var compression = new GzipCompression();

// save the image
N5Utils.save(img, n5Writer, groupPath, blockSize, compression);

In [11]:

var exec = Executors.newFixedThreadPool(4); // with 4 parallel threads
N5Utils.save(img, n5Writer, groupPath, blockSize, compression, exec);

In [12]:

Code

printBlocks("my-container.n5/data");

my-container.n5/data/1/1 is 1762 bytes
my-container.n5/data/1/0 is 2012 bytes
my-container.n5/data/0/1 is 1763 bytes
my-container.n5/data/0/0 is 2020 bytes

In [13]:

// remove the old data
n5Writer.remove(groupPath);

// rewrite with a different block size
var blockSize = new int[]{64,8};
N5Utils.save(img, n5Writer, groupPath, blockSize, compression);

// how many blocks are there?
printBlocks("my-container.n5/data");

my-container.n5/data/0/1 is 837 bytes
my-container.n5/data/0/7 is 847 bytes
my-container.n5/data/0/3 is 839 bytes
my-container.n5/data/0/6 is 844 bytes
my-container.n5/data/0/0 is 968 bytes
my-container.n5/data/0/4 is 846 bytes
my-container.n5/data/0/2 is 840 bytes
my-container.n5/data/0/5 is 847 bytes

In [14]:

// rewrite without compression
var groupPath = "dataNoCompression"; 
var blockSize = new int[]{32,32};
var compression = new RawCompression();
N5Utils.save(img, n5Writer, groupPath, blockSize, compression);

// what size are the blocks?

In [15]:

Code

printBlocks("my-container.n5/dataNoCompression");

my-container.n5/dataNoCompression/1/1 is 4108 bytes
my-container.n5/dataNoCompression/1/0 is 4108 bytes
my-container.n5/dataNoCompression/0/1 is 4108 bytes
my-container.n5/dataNoCompression/0/0 is 4108 bytes

In [16]:

var loadedImg = N5Utils.open(n5Writer, groupPath);
Util.getTypeFromInterval(loadedImg).getClass();      // FloatType
Arrays.toString(loadedImg.dimensionsAsLongArray());  // [64, 64]

In [17]:

// overwrite our previous data
var img = ArrayImgs.unsignedBytes(2,2);
N5Utils.save(img, n5Writer, groupPath, blockSize, compression);

// load the new data, the old data are no longer accessible
var loadedImg = N5Utils.open(n5Writer, groupPath);
Arrays.toString(loadedImg.dimensionsAsLongArray());  // [2, 2]

In [18]:

// create a group inside the container (think: "folder")
var groupName = "put-data-in-me";
n5Writer.createGroup(groupName);

// attributes have names and values
// make an attribute called "date" with a String value
var attributeName = "date";
n5Writer.setAttribute(groupName, attributeName, "2024-Jan-01");

// Ask the N5 API to make a double array from the data attribute
// it will try and fail, so an exception will be thrown
try {
    var nothing = n5Writer.getAttribute(groupName, attributeName, double[].class);
} catch( N5Exception e ) {
    System.out.println("Error: could not get attribute as double[]");
}

// get the value of the "date" attribute as a String
String date = n5Writer.getAttribute(groupName, attributeName, String.class);
date

Error: could not get attribute as double[]

2024-Jan-01

In [19]:

n5Writer.setAttribute(groupName, "a", 42);
var num = n5Writer.getAttribute(groupName, "a", double.class); // 42.0
var str = n5Writer.getAttribute(groupName, "a", String.class); // "42"

In [20]:

Code

class FunWithMetadata {
    String name;
    int number;
    double[] data;
    
    public FunWithMetadata(String name, int number, double[] data) {
        this.name = name;
        this.number = number;
        this.data = data;
    }
    public String toString(){
        return String.format( "FunWithMetadata{%s(%d): %s}", 
            name, number, Arrays.toString(data));
    }
};

In [21]:

var metadata = new FunWithMetadata("Dorothy", 2, new double[]{2.72, 3.14});
n5Writer.setAttribute(groupName, "metadata", metadata);

// get attribute as an instance of FunWithMetdata
n5Writer.getAttribute(groupName, "metadata",  FunWithMetadata.class);

FunWithMetadata{Dorothy(2): [2.72, 3.14]}

In [22]:

// get attribute as an instance of JsonElement
n5Writer.getAttribute(groupName, "/", JsonElement.class);

{"date":"2024-Jan-01","a":42,"metadata":{"name":"Dorothy","number":2,"data":[2.72,3.14]}}

In [23]:

// set attributes
n5Writer.setAttribute(groupName, "sender", "Alice");
n5Writer.setAttribute(groupName, "receiver", "Bob");

// notice that they're set
n5Writer.getAttribute(groupName, "sender", String.class);   // Alice
n5Writer.getAttribute(groupName, "receiver", String.class); // Bob

// remove "sender"
n5Writer.removeAttribute(groupName, "sender");

// remove "receiver" and store result in a variable
var receiver = n5Writer.removeAttribute(groupName, "receiver", String.class); // Bob

n5Writer.getAttribute(groupName, "sender", String.class);   // null
n5Writer.getAttribute(groupName, "receiver", String.class); // null

In [24]:

Arrays.toString(n5Writer.getAttribute("data", "dimensions", long[].class));

[64, 64]

In [25]:

var arrayMetadata = n5Writer.getDatasetAttributes("data");
arrayMetadata.getDimensions();
arrayMetadata.getBlockSize();
arrayMetadata.getDataType();
arrayMetadata.getCompression();

N5-Basics-Tutorial.ipynb