Source code for bioio_base.io
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, Tuple
from urllib.parse import parse_qs, urlparse
from fsspec.core import url_to_fs
if TYPE_CHECKING:
from fsspec.spec import AbstractFileSystem
from .types import ImageLike, PathLike
###############################################################################
[docs]
def pathlike_to_fs(
uri: PathLike,
enforce_exists: bool = False,
fs_kwargs: Dict[str, Any] = {},
) -> Tuple["AbstractFileSystem", str]:
"""
Find and return the appropriate filesystem and path from a path-like object.
Parameters
----------
uri: PathLike
The local or remote path or uri.
enforce_exists: bool
Check whether or not the resource exists, if not, raise FileNotFoundError.
Returns
-------
fs: AbstractFileSystem
The filesystem to operate on.
path: str
The full path to the target resource.
fs_kwargs: Dict[str, Any]
Any specific keyword arguments to pass down to the fsspec created filesystem.
Default: {}
Raises
------
FileNotFoundError
If enforce_exists is provided value True and the resource is not found or is
unavailable.
"""
# Convert paths to string to be handled by url_to_fs
if isinstance(uri, Path):
uri = str(uri)
# Get details
fs, path = url_to_fs(uri, **fs_kwargs)
# Check file exists
if enforce_exists:
if not fs.exists(path):
raise FileNotFoundError(f"{fs.protocol}://{path}") # noqa: E231
# Get and store details
# We do not return an AbstractBufferedFile (i.e. fs.open) as we do not want to have
# any open file buffers _after_ any API call. API calls must themselves call
# fs.open and complete their function during the context of the opened buffer.
return fs, path
[docs]
def s3_https_to_s3(
uri: ImageLike,
fs_kwargs: Dict[str, Any] = {},
) -> Tuple[ImageLike, Dict[str, Any]]:
"""
Rewrite an S3 HTTPS URL as an anonymous ``s3://`` URI.
A directory can't be listed over HTTP(S), but the same objects can be over
``s3://``. Both ``https://s3.<region>.amazonaws.com/<bucket>/<key>`` and
``https://<bucket>.s3.<region>.amazonaws.com/<key>`` are recognized.
Parameters
----------
uri: ImageLike
The image to rewrite. Anything other than an S3 HTTPS URL is returned
unchanged.
fs_kwargs: Dict[str, Any]
Any specific keyword arguments to pass down to the fsspec created filesystem.
Default: {}
Returns
-------
uri: ImageLike
The ``s3://`` URI, or the given uri if it is not an S3 HTTPS URL.
fs_kwargs: Dict[str, Any]
The given fs_kwargs, with ``anon=True`` added for a rewritten URL unless
already set.
Raises
------
ValueError
If the URL is presigned. A signature covers one object, so it can't be
carried over to listing a directory.
"""
if not isinstance(uri, str):
return uri, fs_kwargs
url = urlparse(uri)
if url.scheme not in ("http", "https") or not url.netloc.endswith(".amazonaws.com"):
return uri, fs_kwargs
if url.netloc.startswith("s3."):
bucket_and_key = url.path.lstrip("/")
else:
bucket, virtual_hosted, _ = url.netloc.rpartition(".s3.")
if not virtual_hosted:
return uri, fs_kwargs
bucket_and_key = f"{bucket}{url.path}"
if {"x-amz-signature", "signature"} & {k.lower() for k in parse_qs(url.query)}:
raise ValueError(
f"{uri} is a presigned URL, which only grants access to a single object. "
"Use the s3:// URI or a public S3 URL to read a directory."
)
return "s3://" + bucket_and_key, {"anon": True, **fs_kwargs}