diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d2255c4dd..31aa703389 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -113,9 +113,7 @@ jobs: if: matrix.os == 'ubuntu-latest' id: clone-services run: | - # 2026-11-05: Services switched to SeaweedFS and the MinIO client cannot handle the new URLs. - # We use the old stack until we have migrated away from the MinIO client - git clone --revision dd8b98a9e0450bf487f973e1711c8b00dc022294 --depth 1 https://github.com/openml/services.git + git clone --revision a411948cb41d9d9fd99099e103d7b6871138abf4 --depth 1 https://github.com/openml/services.git - name: Start Docker Services id: start-services diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8aa214a8a6..71fc9bd2f4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,6 +15,7 @@ repos: additional_dependencies: - types-requests - types-python-dateutil + - boto3-stubs[s3] - repo: https://github.com/python-jsonschema/check-jsonschema rev: 0.38.2 hooks: diff --git a/openml/_api/__init__.py b/openml/_api/__init__.py index 7766016d19..a7c1a263cc 100644 --- a/openml/_api/__init__.py +++ b/openml/_api/__init__.py @@ -1,7 +1,7 @@ from .clients import ( HTTPCache, HTTPClient, - MinIOClient, + S3Client, ) from .resources import ( API_REGISTRY, @@ -65,13 +65,13 @@ "FlowV2API", "HTTPCache", "HTTPClient", - "MinIOClient", "ResourceAPI", "ResourceV1API", "ResourceV2API", "RunAPI", "RunV1API", "RunV2API", + "S3Client", "SetupAPI", "SetupV1API", "SetupV2API", diff --git a/openml/_api/clients/__init__.py b/openml/_api/clients/__init__.py index 42f11fbcff..ca2ecab4af 100644 --- a/openml/_api/clients/__init__.py +++ b/openml/_api/clients/__init__.py @@ -1,8 +1,8 @@ from .http import HTTPCache, HTTPClient -from .minio import MinIOClient +from .s3 import S3Client __all__ = [ "HTTPCache", "HTTPClient", - "MinIOClient", + "S3Client", ] diff --git a/openml/_api/clients/minio.py b/openml/_api/clients/minio.py deleted file mode 100644 index 852b678cd0..0000000000 --- a/openml/_api/clients/minio.py +++ /dev/null @@ -1,149 +0,0 @@ -from __future__ import annotations - -import contextlib -import shutil -import urllib -import zipfile -from pathlib import Path - -import minio -import requests -from urllib3 import ProxyManager - -import openml -from openml.utils import ProgressBar - - -class MinIOClient: - """ - Lightweight client configuration for interacting with a MinIO-compatible - object storage service. - - This class stores basic configuration such as a base filesystem path and - default HTTP headers. It is intended to be extended with actual request - or storage logic elsewhere. - - Attributes - ---------- - path : pathlib.Path - Configured base path for storage operations. - headers : dict of str to str - Default HTTP headers, including a user-agent identifying the - OpenML Python client version. - """ - - @property - def headers(self) -> dict[str, str]: - return openml.config._HEADERS - - @property - def path(self) -> Path: - return Path(openml.config.get_cache_directory()) - - def download_minio_file( - self, - source: str, - exists_ok: bool = True, # noqa: FBT002 - proxy: str | None = "auto", - ) -> Path: - """Download file ``source`` from a MinIO Bucket and store it at ``destination``. - - Parameters - ---------- - source : str - URL to a file in a MinIO bucket. - exists_ok : bool, optional (default=True) - If False, raise FileExists if a file already exists in ``destination``. - proxy: str, optional (default = "auto") - The proxy server to use. By default it's "auto" which uses ``requests`` to - automatically find the proxy to use. Pass None or the environment variable - ``no_proxy="*"`` to disable proxies. - """ - destination = Path(openml.config.get_minio_download_path(source)) - parsed_url = urllib.parse.urlparse(source) - - # expect path format: /BUCKET/path/to/file.ext - bucket, object_name = parsed_url.path[1:].split("/", maxsplit=1) - if destination.is_dir(): - destination = Path(destination, object_name) - if destination.is_file() and not exists_ok: - raise FileExistsError(f"File already exists in {destination}.") - - destination = destination.expanduser() - destination.parent.mkdir(parents=True, exist_ok=True) - - if proxy == "auto": - resolved_proxies = requests.utils.get_environ_proxies(parsed_url.geturl()) - proxy = requests.utils.select_proxy(parsed_url.geturl(), resolved_proxies) # type: ignore - - proxy_client = ProxyManager(proxy) if proxy else None - - client = minio.Minio(endpoint=parsed_url.netloc, secure=False, http_client=proxy_client) - try: - client.fget_object( - bucket_name=bucket, - object_name=object_name, - file_path=str(destination), - progress=ProgressBar() if openml.config.show_progress else None, - request_headers=self.headers, - ) - if destination.is_file() and destination.suffix == ".zip": - with zipfile.ZipFile(destination, "r") as zip_ref: - zip_ref.extractall(destination.parent) - - except minio.error.S3Error as e: - if e.message is not None and e.message.startswith("Object does not exist"): - raise FileNotFoundError(f"Object at '{source}' does not exist.") from e - # e.g. permission error, or a bucket does not exist (which is also interpreted as a - # permission error on minio level). - raise FileNotFoundError("Bucket does not exist or is private.") from e - - return destination - - def download_minio_bucket(self, source: str) -> None: - """Download file ``source`` from a MinIO Bucket and store it at ``destination``. - - Does not redownload files which already exist. - - Parameters - ---------- - source : str - URL to a MinIO bucket. - """ - destination = Path(openml.config.get_minio_download_path(source)) - parsed_url = urllib.parse.urlparse(source) - if destination.suffix: - destination = destination.parent - # expect path format: /BUCKET/path/to/file.ext - _, bucket, *prefixes, _ = parsed_url.path.split("/") - prefix = "/".join(prefixes) - - client = minio.Minio(endpoint=parsed_url.netloc, secure=False) - - for file_object in client.list_objects(bucket, prefix=prefix, recursive=True): - if file_object.object_name is None: - raise ValueError(f"Object name is None for object {file_object!r}") - if file_object.etag is None: - raise ValueError(f"Object etag is None for object {file_object!r}") - - marker = destination / file_object.etag - if marker.exists(): - continue - - file_destination = destination / file_object.object_name.rsplit("/", 1)[1] - if (file_destination.parent / file_destination.stem).exists(): - # Marker is missing but archive exists means the server archive changed - # force a refresh - shutil.rmtree(file_destination.parent / file_destination.stem) - - with contextlib.suppress(FileExistsError): - self.download_minio_file( - source=source.rsplit("/", 1)[0] - + "/" - + file_object.object_name.rsplit("/", 1)[1], - exists_ok=False, - ) - - if file_destination.is_file() and file_destination.suffix == ".zip": - file_destination.unlink() - marker.touch() diff --git a/openml/_api/clients/s3.py b/openml/_api/clients/s3.py new file mode 100644 index 0000000000..66cb711664 --- /dev/null +++ b/openml/_api/clients/s3.py @@ -0,0 +1,229 @@ +from __future__ import annotations + +import contextlib +import shutil +import urllib +import zipfile +from pathlib import Path +from typing import TYPE_CHECKING + +import boto3 +import requests +from botocore import UNSIGNED +from botocore.config import Config +from botocore.exceptions import ClientError + +if TYPE_CHECKING: + from mypy_boto3_s3.client import S3Client as BotoS3Client + +import openml +from openml.utils import ProgressBar + + +class S3Client: + """ + Lightweight client configuration for interacting with an S3-compatible + object storage service. + + This class stores basic configuration such as a base filesystem path and + default HTTP headers. It is intended to be extended with actual request + or storage logic elsewhere. + """ + + @staticmethod + def _parse_s3_url( + source: str, + ) -> tuple[str, str, str]: + """Split an object storage URL into endpoint, bucket, and object key. + + The optional mount prefix is kept on the + endpoint URL so path-style S3 requests still go through the reverse + proxy. Production URLs have no mount prefix. + + Parameters + ---------- + source : str + URL to a file in an S3 bucket, for example: + ``http://localhost:8000/s3/datasets/0000/0061/dataset_61.pq`` + + Returns + ------- + tuple[str, str, str] + Endpoint URL, bucket name, object key. + + Raises + ------ + ValueError + If the URL does not contain both a bucket and an object key. + """ + # Parse the URL into sections + parsed_url = urllib.parse.urlparse(source) + parts = [part for part in parsed_url.path.split("/") if part] + if len(parts) < 2: # Bucket and key minimum + raise ValueError(f"Object storage URL must contain a bucket and object key: {source!r}") + + # Handle the endpoint + endpoint = f"{parsed_url.scheme}://{parsed_url.netloc}" + if parts[0] == "s3": # URL has mount in first index + endpoint += f"/{parts.pop(0)}" + + # Handle the bucket and object key + bucket = parts[0] + object_key = "/".join(parts[1:]) + if not object_key: + raise ValueError(f"Object storage URL must contain an object key: {source!r}.") + return endpoint, bucket, object_key + + @staticmethod + def _make_s3_client( + endpoint_url: str, + proxy: str | None = None, + ) -> BotoS3Client: + """Create an anonymous S3 client for a public object-storage endpoint. + + Parameters + ---------- + endpoint_url : str + Endpoint URL, including any reverse-proxy mount prefix. + proxy : str or None, optional (default=None) + Proxy URL used for both HTTP and HTTPS requests. ``None`` means + a direct connection. + + Returns + ------- + botocore.client.BotoS3Client + Configured S3 client. + """ + proxies = {"http": proxy, "https": proxy} if proxy else None + config = Config( + signature_version=UNSIGNED, # Public bucket + s3={"addressing_style": "path"}, # URL = endpoint/bucket/key for example + proxies=proxies, # Optional HTTP proxy + user_agent_extra=openml.config._HEADERS["user-agent"], # Keep openml-python/ + ) + return boto3.client( + "s3", + endpoint_url=endpoint_url, + region_name="eu-central-1", + config=config, + ) + + def download_file( + self, + source: str, + exists_ok: bool = True, # noqa: FBT002 + proxy: str | None = "auto", + ) -> Path: + """Download file ``source`` from an S3 Bucket and store it at ``destination``. + + Parameters + ---------- + source : str + URL to a file in an S3 bucket. + exists_ok : bool, optional (default=True) + If False, raise FileExists if a file already exists in ``destination``. + proxy: str, optional (default = "auto") + Proxy server to use. ``"auto"`` resolves it from the environment. + """ + # Parse the source url and get cache location + endpoint, bucket, object_key = self._parse_s3_url(source) + destination = Path(openml.config.get_download_path(source)) + + # Ensure the cache does not already exist + if destination.is_dir(): + destination = Path(destination, object_key.rsplit("/", 1)[-1]) + if destination.is_file() and not exists_ok: + raise FileExistsError(f"File already exists in {destination}.") + + # Ensure download location exists + destination = destination.expanduser() + destination.parent.mkdir(parents=True, exist_ok=True) + + # Set up the S3 client with the proxy + if proxy == "auto": + resolved_proxies = requests.utils.get_environ_proxies(source) + proxy = requests.utils.select_proxy(source, resolved_proxies) + client = self._make_s3_client(endpoint_url=endpoint, proxy=proxy) + + # Get the download object + try: + # Create a progress bar if required + callback = None + if openml.config.show_progress: + total_length = client.head_object(Bucket=bucket, Key=object_key)["ContentLength"] + progress = ProgressBar() + progress.set_meta(object_key, total_length) + callback = progress.update + # Download object + client.download_file( + Bucket=bucket, + Key=object_key, + Filename=str(destination), + Callback=callback, + ) + # Unzip if compressed + if destination.is_file() and destination.suffix == ".zip": + with zipfile.ZipFile(destination, "r") as zip_ref: + zip_ref.extractall(destination.parent) + + # Error handling and return + except ClientError as err: + error = err.response.get("Error", {}) + status = err.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if status == 404 or error.get("Code") in {"404", "NoSuchKey", "NotFound"}: + raise FileNotFoundError(f"Object at '{source}' does not exist.") from err + raise FileNotFoundError("Bucket is private or does not exist.") from err + except RecursionError as err: + raise RuntimeError( + f"Redirect loop while fetching {source!r}. " + "The endpoint likely redirects HTTP to HTTPS; use an https:// URL." + ) from err + return destination + + def download_bucket(self, source: str) -> None: + """Download files from S3 Bucket ``source`` and store it at ``destination``. + + Does not redownload files which already exist. + + Parameters + ---------- + source : str + URL to a S3 bucket. + """ + # Parse the source information + endpoint, bucket, object_key = self._parse_s3_url(source) + destination = Path(openml.config.get_download_path(source)) + if destination.suffix: + destination = destination.parent + prefix = object_key.rsplit("/", 1)[0] + + # Create S3 client and paginator + client = self._make_s3_client(endpoint_url=endpoint) + paginator = client.get_paginator("list_objects_v2") # PageSize caps at 1_000 + + # Iterate over the response pages + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + for file_object in page.get("Contents", []): + obj_key = file_object["Key"] + etag = (file_object.get("ETag") or "").strip('"') + + # Check if the object was already downloaded + marker = destination / etag if etag else None + if marker is not None and marker.exists(): + continue + + # Server archive changed? + object_source = f"{endpoint}/{bucket}/{obj_key}" + file_destination = Path(openml.config.get_download_path(object_source)) + if (file_destination.parent / file_destination.stem).exists(): + shutil.rmtree(file_destination.parent / file_destination.stem) + + # No need to redownload file + with contextlib.suppress(FileExistsError): + self.download_file(source=object_source, exists_ok=False) + + # Clean up unzipped file then leave marker + if file_destination.is_file() and file_destination.suffix == ".zip": + file_destination.unlink() + if marker is not None: + marker.touch() diff --git a/openml/_api/resources/base/base.py b/openml/_api/resources/base/base.py index 625681e3b2..b6bc98cb0a 100644 --- a/openml/_api/resources/base/base.py +++ b/openml/_api/resources/base/base.py @@ -14,7 +14,7 @@ from collections.abc import Mapping from typing import Any - from openml._api.clients import HTTPClient, MinIOClient + from openml._api.clients import HTTPClient, S3Client from openml.enums import APIVersion, ResourceType @@ -31,8 +31,8 @@ class ResourceAPI(ABC): ---------- http : HTTPClient Configured HTTP client used for communication with the OpenML API. - minio : MinIOClient - Configured MinIO client used for object storage operations. + s3 : S3Client + Configured S3 client used for object storage operations. Attributes ---------- @@ -42,16 +42,16 @@ class ResourceAPI(ABC): Type of OpenML resource handled by the implementation. _http : HTTPClient Internal HTTP client instance. - _minio : MinIOClient or None - Internal MinIO client instance, if provided. + _s3 : S3Client or None + Internal S3 client instance, if provided. """ api_version: APIVersion resource_type: ResourceType - def __init__(self, http: HTTPClient, minio: MinIOClient): + def __init__(self, http: HTTPClient, s3: S3Client): self._http = http - self._minio = minio + self._s3 = s3 @abstractmethod def delete(self, resource_id: int) -> bool: diff --git a/openml/_api/resources/dataset.py b/openml/_api/resources/dataset.py index c69f3fbdc8..207f838f30 100644 --- a/openml/_api/resources/dataset.py +++ b/openml/_api/resources/dataset.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Literal -import minio +import botocore import pandas as pd import urllib3 import xmltodict @@ -760,19 +760,19 @@ def download_dataset_parquet( else: raise TypeError("`description` should be either OpenMLDataset or Dict.") - output_file_path = Path(openml.config.get_minio_download_path(url)) + output_file_path = Path(openml.config.get_download_path(url)) if download_all_files: - self._minio.download_minio_bucket(source=url) + self._s3.download_bucket(source=url) if not output_file_path.is_file(): try: - output_file_path = self._minio.download_minio_file( + output_file_path = self._s3.download_file( source=url, ) except ( FileNotFoundError, urllib3.exceptions.MaxRetryError, - minio.error.ServerError, + botocore.exceptions.BotoCoreError, ) as e: logger.warning(f"Could not download file from {url}: {e}") return None @@ -1501,17 +1501,17 @@ def download_dataset_parquet( else: raise TypeError("`description` should be either OpenMLDataset or Dict.") - output_file_path = Path(openml.config.get_minio_download_path(url)) + output_file_path = Path(openml.config.get_download_path(url)) if download_all_files: - self._minio.download_minio_bucket(source=url) + self._s3.download_bucket(source=url) if not output_file_path.is_file(): try: - output_file_path = self._minio.download_minio_file(source=url) + output_file_path = self._s3.download_file(source=url) except ( FileNotFoundError, urllib3.exceptions.MaxRetryError, - minio.error.ServerError, + botocore.exceptions.BotoCoreError, ) as e: logger.warning(f"Could not download file from {url}: {e}") return None diff --git a/openml/_api/setup/backend.py b/openml/_api/setup/backend.py index 1604fd074a..684c7cff17 100644 --- a/openml/_api/setup/backend.py +++ b/openml/_api/setup/backend.py @@ -7,7 +7,7 @@ from .builder import APIBackendBuilder if TYPE_CHECKING: - from openml._api.clients import HTTPClient, MinIOClient + from openml._api.clients import HTTPClient, S3Client from openml._api.resources import ( DatasetAPI, EstimationProcedureAPI, @@ -121,8 +121,8 @@ def fallback_http_client(self) -> HTTPClient | None: return cast("HTTPClient | None", self._backend.fallback_http_client) @property - def minio_client(self) -> MinIOClient: - return cast("MinIOClient", self._backend.minio_client) + def s3_client(self) -> S3Client: + return cast("S3Client", self._backend.s3_client) @classmethod def get_instance(cls) -> APIBackend: diff --git a/openml/_api/setup/builder.py b/openml/_api/setup/builder.py index 76d6e09705..5387919e83 100644 --- a/openml/_api/setup/builder.py +++ b/openml/_api/setup/builder.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -from openml._api.clients import HTTPClient, MinIOClient +from openml._api.clients import HTTPClient, S3Client from openml._api.resources import ( API_REGISTRY, FallbackProxy, @@ -27,7 +27,7 @@ class APIBackendBuilder: - HTTPClient for the primary API version - Optional HTTPClient for a fallback API version - - MinIOClient for file storage operations + - S3Client for file storage operations - Resource-specific API instances, optionally wrapped with fallback proxies Parameters @@ -61,8 +61,8 @@ class APIBackendBuilder: Client for HTTP communication using the primary API version. fallback_http_client : HTTPClient | None Client for HTTP communication using the fallback API version, if provided. - minio_client : MinIOClient - Client for file storage operations (MinIO/S3). + s3_client : S3Client + Client for file storage operations (S3). """ dataset: ResourceAPI | FallbackProxy @@ -76,7 +76,7 @@ class APIBackendBuilder: setup: ResourceAPI | FallbackProxy http_client: HTTPClient fallback_http_client: HTTPClient | None - minio_client: MinIOClient + s3_client: S3Client def __init__(self, api_version: APIVersion, fallback_api_version: APIVersion | None = None): # initialize clients and resource APIs in-place @@ -86,7 +86,7 @@ def _build(self, api_version: APIVersion, fallback_api_version: APIVersion | Non """ Construct an APIBackendBuilder instance from a configuration. - This method initializes HTTP and MinIO clients, creates resource-specific + This method initializes HTTP and S3 clients, creates resource-specific API instances for the primary API version, and optionally wraps them with fallback proxies if a fallback API version is configured. @@ -101,16 +101,16 @@ def _build(self, api_version: APIVersion, fallback_api_version: APIVersion | Non APIBackendBuilder Builder instance with all resource API interfaces initialized. """ - minio_client = MinIOClient() + s3_client = S3Client() primary_http_client = HTTPClient(api_version=api_version) self.http_client = primary_http_client - self.minio_client = minio_client + self.s3_client = s3_client self.fallback_http_client = None resource_apis: dict[ResourceType, ResourceAPI | FallbackProxy] = {} for resource_type, resource_api_cls in API_REGISTRY[api_version].items(): - resource_apis[resource_type] = resource_api_cls(primary_http_client, minio_client) + resource_apis[resource_type] = resource_api_cls(primary_http_client, s3_client) if fallback_api_version is not None: fallback_http_client = HTTPClient(api_version=fallback_api_version) @@ -119,7 +119,7 @@ def _build(self, api_version: APIVersion, fallback_api_version: APIVersion | Non fallback_resource_apis: dict[ResourceType, ResourceAPI | FallbackProxy] = {} for resource_type, resource_api_cls in API_REGISTRY[fallback_api_version].items(): fallback_resource_apis[resource_type] = resource_api_cls( - fallback_http_client, minio_client + fallback_http_client, s3_client ) resource_apis = { diff --git a/openml/_api_calls.py b/openml/_api_calls.py index aee7c9b10c..4659973687 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -1,24 +1,18 @@ # License: BSD 3-Clause from __future__ import annotations -import contextlib import hashlib import logging import math import random -import shutil import time -import urllib.parse import xml.parsers.expat -import zipfile from pathlib import Path from typing import cast -import minio import requests import requests.utils import xmltodict -from urllib3 import ProxyManager import openml @@ -30,7 +24,6 @@ OpenMLServerException, OpenMLServerNoResult, ) -from .utils import ProgressBar _HEADERS = {"user-agent": f"openml-python/{__version__}"} @@ -131,111 +124,6 @@ def _perform_api_call( return response.text -def _download_minio_file( - source: str, - destination: str | Path, - exists_ok: bool = True, # noqa: FBT002 - proxy: str | None = "auto", -) -> None: - """Download file ``source`` from a MinIO Bucket and store it at ``destination``. - - Parameters - ---------- - source : str - URL to a file in a MinIO bucket. - destination : str | Path - Path to store the file to, if a directory is provided the original filename is used. - exists_ok : bool, optional (default=True) - If False, raise FileExists if a file already exists in ``destination``. - proxy: str, optional (default = "auto") - The proxy server to use. By default it's "auto" which uses ``requests`` to - automatically find the proxy to use. Pass None or the environment variable - ``no_proxy="*"`` to disable proxies. - """ - destination = Path(destination) - parsed_url = urllib.parse.urlparse(source) - - # expect path format: /BUCKET/path/to/file.ext - bucket, object_name = parsed_url.path[1:].split("/", maxsplit=1) - if destination.is_dir(): - destination = Path(destination, object_name) - if destination.is_file() and not exists_ok: - raise FileExistsError(f"File already exists in {destination}.") - - if proxy == "auto": - proxy = resolve_env_proxies(parsed_url.geturl()) - - proxy_client = ProxyManager(proxy) if proxy else None - - client = minio.Minio(endpoint=parsed_url.netloc, secure=False, http_client=proxy_client) - try: - client.fget_object( - bucket_name=bucket, - object_name=object_name, - file_path=str(destination), - progress=ProgressBar() if openml.config.show_progress else None, - request_headers=_HEADERS, - ) - if destination.is_file() and destination.suffix == ".zip": - with zipfile.ZipFile(destination, "r") as zip_ref: - zip_ref.extractall(destination.parent) - - except minio.error.S3Error as e: - if e.message is not None and e.message.startswith("Object does not exist"): - raise FileNotFoundError(f"Object at '{source}' does not exist.") from e - # e.g. permission error, or a bucket does not exist (which is also interpreted as a - # permission error on minio level). - raise FileNotFoundError("Bucket does not exist or is private.") from e - - -def _download_minio_bucket(source: str, destination: str | Path) -> None: - """Download file ``source`` from a MinIO Bucket and store it at ``destination``. - - Does not redownload files which already exist. - - Parameters - ---------- - source : str - URL to a MinIO bucket. - destination : str | Path - Path to a directory to store the bucket content in. - """ - destination = Path(destination) - parsed_url = urllib.parse.urlparse(source) - - # expect path format: /BUCKET/path/to/file.ext - _, bucket, *prefixes, _file = parsed_url.path.split("/") - prefix = "/".join(prefixes) - - client = minio.Minio(endpoint=parsed_url.netloc, secure=False) - - for file_object in client.list_objects(bucket, prefix=prefix, recursive=True): - if file_object.object_name is None: - raise ValueError(f"Object name is None for object {file_object!r}") - if file_object.etag is None: - raise ValueError(f"Object etag is None for object {file_object!r}") - - marker = destination / file_object.etag - if marker.exists(): - continue - - file_destination = destination / file_object.object_name.rsplit("/", 1)[1] - if (file_destination.parent / file_destination.stem).exists(): - # Marker is missing but archive exists means the server archive changed, force a refresh - shutil.rmtree(file_destination.parent / file_destination.stem) - - with contextlib.suppress(FileExistsError): - _download_minio_file( - source=source.rsplit("/", 1)[0] + "/" + file_object.object_name.rsplit("/", 1)[1], - destination=file_destination, - exists_ok=False, - ) - - if file_destination.is_file() and file_destination.suffix == ".zip": - file_destination.unlink() - marker.touch() - - def _download_text_file( source: str, output_path: str | Path | None = None, diff --git a/openml/_config.py b/openml/_config.py index c48dfd153c..12d5df0530 100644 --- a/openml/_config.py +++ b/openml/_config.py @@ -500,9 +500,9 @@ def get_cache_directory(self, server_url: str | None = None) -> str: reversed_url_suffix = os.sep.join(url_parts) # noqa: PTH118 return os.path.join(self._root_cache_directory, reversed_url_suffix) # noqa: PTH118 - def get_minio_download_path(self, url: str) -> str: + def get_download_path(self, url: str) -> str: parsed_url = urlparse(url) - return os.path.join(self.get_cache_directory(url), "minio", parsed_url.path.lstrip("/")) # noqa: PTH118 + return os.path.join(self.get_cache_directory(url), "s3", parsed_url.path.lstrip("/")) # noqa: PTH118 def set_root_cache_directory(self, root_cache_directory: str | Path) -> None: """Set the root cache directory.""" diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index b87e5cee0d..5b347bea22 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -117,7 +117,7 @@ class OpenMLDataset(OpenMLBase): # noqa: PLW1641, RUF100 Serialized arff dataset string. parquet_url: string, optional This is the URL to the storage location where the dataset files are hosted. - This can be a MinIO bucket URL. If specified, the data will be accessed + This can be an S3 bucket URL. If specified, the data will be accessed from this URL when reading the files. parquet_file: string, optional Path to the local file. diff --git a/openml/testing.py b/openml/testing.py index 0489509347..774a59ca39 100644 --- a/openml/testing.py +++ b/openml/testing.py @@ -15,7 +15,7 @@ import requests import openml -from openml._api import HTTPClient, MinIOClient +from openml._api import HTTPClient, S3Client from openml.enums import APIVersion from openml.exceptions import OpenMLServerException from openml.tasks import TaskType @@ -56,7 +56,7 @@ class TestBase(unittest.TestCase): logger.setLevel(logging.DEBUG) http_client: HTTPClient = HTTPClient(api_version=APIVersion.V1) - minio_client = MinIOClient() + s3_client = S3Client() def setUp(self, n_levels: int = 1, tmpdir_suffix: str = "") -> None: """Setup variables and temporary directories. diff --git a/openml/utils/_openml.py b/openml/utils/_openml.py index 125dcd08ba..cd997e880a 100644 --- a/openml/utils/_openml.py +++ b/openml/utils/_openml.py @@ -20,7 +20,6 @@ import numpy as np import xmltodict -from minio.helpers import ProgressType from tqdm import tqdm import openml @@ -456,8 +455,8 @@ def _create_lockfiles_dir() -> Path: return path -class ProgressBar(ProgressType): - """Progressbar for MinIO function's `progress` parameter.""" +class ProgressBar: + """Progressbar for object-storage downloads.""" def __init__(self) -> None: self._object_name = "" diff --git a/pyproject.toml b/pyproject.toml index 460ae37e82..bb9605e6ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,9 +17,9 @@ dependencies = [ "pandas>=1.0.0", "scipy>=0.13.3", "numpy>=1.6.2", - "minio", + "boto3>=1.34", "pyarrow", - "tqdm", # For MinIO download progress bars + "tqdm", # For download progress bars ] requires-python = ">=3.10,<3.15" maintainers = [ @@ -82,6 +82,7 @@ test=[ "packaging", "pytest-mock", "openml-sklearn", + "boto3-stubs[s3]", ] examples=[ "matplotlib", @@ -357,6 +358,7 @@ module = [ "tests.test_api.test_flow", "tests.test_api.test_http", "tests.test_api.test_run", + "tests.test_api.test_s3", "tests.test_api.test_setup", "tests.test_api.test_study", "tests.test_api.test_task", diff --git a/tests/conftest.py b/tests/conftest.py index ffce2887bd..0a60cfe7d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,7 +36,7 @@ import pytest # noqa: E402 import openml # noqa: E402 -from openml._api import HTTPClient, MinIOClient # noqa: E402 +from openml._api import HTTPClient, S3Client # noqa: E402 from openml.enums import APIVersion # noqa: E402 from openml.testing import TestBase # noqa: E402 @@ -359,5 +359,5 @@ def http_client_v2() -> HTTPClient: @pytest.fixture -def minio_client() -> MinIOClient: - return MinIOClient() +def s3_client() -> S3Client: + return S3Client() diff --git a/tests/test_api/test_datasets.py b/tests/test_api/test_datasets.py index e104f3fb00..8aaf28963c 100644 --- a/tests/test_api/test_datasets.py +++ b/tests/test_api/test_datasets.py @@ -20,13 +20,13 @@ @pytest.fixture -def dataset_v1(http_client_v1, minio_client) -> DatasetV1API: - return DatasetV1API(http=http_client_v1, minio=minio_client) +def dataset_v1(http_client_v1, s3_client) -> DatasetV1API: + return DatasetV1API(http=http_client_v1, s3=s3_client) @pytest.fixture -def dataset_v2(http_client_v2, minio_client) -> DatasetV2API: - return DatasetV2API(http=http_client_v2, minio=minio_client) +def dataset_v2(http_client_v2, s3_client) -> DatasetV2API: + return DatasetV2API(http=http_client_v2, s3=s3_client) def _wait_for_dataset_being_processed(dataset, did, status="active", n_tries=10, wait_time=10): diff --git a/tests/test_api/test_estimation_procedure.py b/tests/test_api/test_estimation_procedure.py index fd5c792487..e0a8bad34f 100644 --- a/tests/test_api/test_estimation_procedure.py +++ b/tests/test_api/test_estimation_procedure.py @@ -9,13 +9,13 @@ @pytest.fixture -def estimation_procedure_v1(http_client_v1, minio_client) -> EstimationProcedureV1API: - return EstimationProcedureV1API(http=http_client_v1, minio=minio_client) +def estimation_procedure_v1(http_client_v1, s3_client) -> EstimationProcedureV1API: + return EstimationProcedureV1API(http=http_client_v1, s3=s3_client) @pytest.fixture -def estimation_procedure_v2(http_client_v2, minio_client) -> EstimationProcedureV2API: - return EstimationProcedureV2API(http=http_client_v2, minio=minio_client) +def estimation_procedure_v2(http_client_v2, s3_client) -> EstimationProcedureV2API: + return EstimationProcedureV2API(http=http_client_v2, s3=s3_client) @pytest.mark.test_server diff --git a/tests/test_api/test_evaluation.py b/tests/test_api/test_evaluation.py index 25ef942372..0e2f5af943 100644 --- a/tests/test_api/test_evaluation.py +++ b/tests/test_api/test_evaluation.py @@ -9,13 +9,13 @@ @pytest.fixture -def evaluation_v1(http_client_v1, minio_client) -> EvaluationV1API: - return EvaluationV1API(http=http_client_v1, minio=minio_client) +def evaluation_v1(http_client_v1, s3_client) -> EvaluationV1API: + return EvaluationV1API(http=http_client_v1, s3=s3_client) @pytest.fixture -def evaluation_v2(http_client_v2, minio_client) -> EvaluationV2API: - return EvaluationV2API(http=http_client_v2, minio=minio_client) +def evaluation_v2(http_client_v2, s3_client) -> EvaluationV2API: + return EvaluationV2API(http=http_client_v2, s3=s3_client) @pytest.mark.test_server diff --git a/tests/test_api/test_evaluation_measure.py b/tests/test_api/test_evaluation_measure.py index 3835c7a5fb..92acfe24db 100644 --- a/tests/test_api/test_evaluation_measure.py +++ b/tests/test_api/test_evaluation_measure.py @@ -7,13 +7,13 @@ @pytest.fixture -def evaluation_measure_v1(http_client_v1, minio_client) -> EvaluationMeasureV1API: - return EvaluationMeasureV1API(http=http_client_v1, minio=minio_client) +def evaluation_measure_v1(http_client_v1, s3_client) -> EvaluationMeasureV1API: + return EvaluationMeasureV1API(http=http_client_v1, s3=s3_client) @pytest.fixture -def evaluation_measure_v2(http_client_v2, minio_client) -> EvaluationMeasureV2API: - return EvaluationMeasureV2API(http=http_client_v2, minio=minio_client) +def evaluation_measure_v2(http_client_v2, s3_client) -> EvaluationMeasureV2API: + return EvaluationMeasureV2API(http=http_client_v2, s3=s3_client) @pytest.mark.test_server diff --git a/tests/test_api/test_flow.py b/tests/test_api/test_flow.py index 6c9d440aa7..c3119f930c 100644 --- a/tests/test_api/test_flow.py +++ b/tests/test_api/test_flow.py @@ -15,13 +15,13 @@ @pytest.fixture -def flow_v1(http_client_v1, minio_client) -> FlowV1API: - return FlowV1API(http=http_client_v1, minio=minio_client) +def flow_v1(http_client_v1, s3_client) -> FlowV1API: + return FlowV1API(http=http_client_v1, s3=s3_client) @pytest.fixture -def flow_v2(http_client_v2, minio_client) -> FlowV2API: - return FlowV2API(http=http_client_v2, minio=minio_client) +def flow_v2(http_client_v2, s3_client) -> FlowV2API: + return FlowV2API(http=http_client_v2, s3=s3_client) def _validate_flow(flow: OpenMLFlow) -> None: diff --git a/tests/test_api/test_run.py b/tests/test_api/test_run.py index df7a18520b..1395b41832 100644 --- a/tests/test_api/test_run.py +++ b/tests/test_api/test_run.py @@ -13,13 +13,13 @@ @pytest.fixture -def run_v1(http_client_v1, minio_client) -> RunV1API: - return RunV1API(http=http_client_v1, minio=minio_client) +def run_v1(http_client_v1, s3_client) -> RunV1API: + return RunV1API(http=http_client_v1, s3=s3_client) @pytest.fixture -def run_v2(http_client_v2, minio_client) -> RunV2API: - return RunV2API(http=http_client_v2, minio=minio_client) +def run_v2(http_client_v2, s3_client) -> RunV2API: + return RunV2API(http=http_client_v2, s3=s3_client) def _assert_run_shape(run: OpenMLRun) -> None: diff --git a/tests/test_api/test_s3.py b/tests/test_api/test_s3.py new file mode 100644 index 0000000000..7ba07d3a9b --- /dev/null +++ b/tests/test_api/test_s3.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import pytest +from botocore import UNSIGNED + +from openml._api.clients.s3 import S3Client + +_ENDPOINT = "http://localhost:8000/s3" + + +def test_make_s3_client(mocker) -> None: + boto_client = mocker.patch("openml._api.clients.s3.boto3.client") + + S3Client._make_s3_client(_ENDPOINT) + + kwargs = boto_client.call_args.kwargs + assert kwargs["endpoint_url"] == _ENDPOINT + assert kwargs["region_name"] == "eu-central-1" + assert kwargs["config"].signature_version == UNSIGNED + assert kwargs["config"].s3 == {"addressing_style": "path"} + + +@pytest.mark.parametrize( + "proxy", + [ + "http://proxy.example.com:443", + "https://proxy.example.com:8443", + ], +) +def test_make_s3_client_with_proxy(mocker, proxy: str) -> None: + boto_client = mocker.patch("openml._api.clients.s3.boto3.client") + + S3Client._make_s3_client(_ENDPOINT, proxy=proxy) + + assert boto_client.call_args.kwargs["config"].proxies == { + "http": proxy, + "https": proxy, + } + + +@pytest.mark.parametrize( + ("url", "expected"), + [ + # local reverse-proxy mount + ( + "http://localhost:8000/s3/datasets/0020/dataset_20.pq", + ("http://localhost:8000/s3", "datasets", "0020/dataset_20.pq"), + ), + # production: no mount + ( + "https://data.openml.org/datasets/0020/dataset_20.pq", + ("https://data.openml.org", "datasets", "0020/dataset_20.pq"), + ), + # legacy bucket-first layout + ( + "http://data.openml.org/dataset20/dataset_20.pq", + ("http://data.openml.org", "dataset20", "dataset_20.pq"), + ), + ], +) +def test_parse_s3_url(url: str, expected: tuple[str, str, str]) -> None: + assert S3Client._parse_s3_url(url) == expected + + +@pytest.mark.parametrize( + "url", + [ + "http://localhost:8000/", # no bucket, no key + "http://localhost:8000/s3/datasets", # mount, no key + ], +) +def test_parse_s3_url_rejects_incomplete(url: str) -> None: + with pytest.raises(ValueError, match="object key"): + S3Client._parse_s3_url(url) diff --git a/tests/test_api/test_setup.py b/tests/test_api/test_setup.py index 1f3b03ac6d..2969e0301c 100644 --- a/tests/test_api/test_setup.py +++ b/tests/test_api/test_setup.py @@ -24,13 +24,13 @@ def get_sentinel(): @pytest.fixture -def setup_v1(http_client_v1, minio_client) -> SetupV1API: - return SetupV1API(http=http_client_v1, minio=minio_client) +def setup_v1(http_client_v1, s3_client) -> SetupV1API: + return SetupV1API(http=http_client_v1, s3=s3_client) @pytest.fixture -def setup_v2(http_client_v2, minio_client) -> SetupV2API: - return SetupV2API(http=http_client_v2, minio=minio_client) +def setup_v2(http_client_v2, s3_client) -> SetupV2API: + return SetupV2API(http=http_client_v2, s3=s3_client) @pytest.mark.test_server diff --git a/tests/test_api/test_study.py b/tests/test_api/test_study.py index 7aa34661ce..41e3b3f6b4 100644 --- a/tests/test_api/test_study.py +++ b/tests/test_api/test_study.py @@ -13,15 +13,15 @@ @pytest.fixture -def study_v1(http_client_v1, minio_client) -> StudyV1API: +def study_v1(http_client_v1, s3_client) -> StudyV1API: """Fixture for V1 Study API instance.""" - return StudyV1API(http=http_client_v1, minio=minio_client) + return StudyV1API(http=http_client_v1, s3=s3_client) @pytest.fixture -def study_v2(http_client_v2, minio_client) -> StudyV2API: +def study_v2(http_client_v2, s3_client) -> StudyV2API: """Fixture for V2 Study API instance.""" - return StudyV2API(http=http_client_v2, minio=minio_client) + return StudyV2API(http=http_client_v2, s3=s3_client) def test_v1_list(study_v1, test_server_v1, test_apikey_v1): # noqa: ARG001 diff --git a/tests/test_api/test_task.py b/tests/test_api/test_task.py index 4d30e27403..9ce101936d 100644 --- a/tests/test_api/test_task.py +++ b/tests/test_api/test_task.py @@ -20,13 +20,13 @@ @pytest.fixture -def task_v1(http_client_v1, minio_client) -> TaskV1API: - return TaskV1API(http=http_client_v1, minio=minio_client) +def task_v1(http_client_v1, s3_client) -> TaskV1API: + return TaskV1API(http=http_client_v1, s3=s3_client) @pytest.fixture -def task_v2(http_client_v2, minio_client) -> TaskV2API: - return TaskV2API(http=http_client_v2, minio=minio_client) +def task_v2(http_client_v2, s3_client) -> TaskV2API: + return TaskV2API(http=http_client_v2, s3=s3_client) def _assert_task_1(task): diff --git a/tests/test_api/test_versions.py b/tests/test_api/test_versions.py index fc655b6e96..8f83f6ae76 100644 --- a/tests/test_api/test_versions.py +++ b/tests/test_api/test_versions.py @@ -24,13 +24,13 @@ class DummyTaskV2API(ResourceV2API, DummyTaskAPI): @pytest.fixture -def dummy_task_v1(http_client_v1, minio_client) -> DummyTaskV1API: - return DummyTaskV1API(http=http_client_v1, minio=minio_client) +def dummy_task_v1(http_client_v1, s3_client) -> DummyTaskV1API: + return DummyTaskV1API(http=http_client_v1, s3=s3_client) @pytest.fixture -def dummy_task_v2(http_client_v2, minio_client) -> DummyTaskV1API: - return DummyTaskV2API(http=http_client_v2, minio=minio_client) +def dummy_task_v2(http_client_v2, s3_client) -> DummyTaskV1API: + return DummyTaskV2API(http=http_client_v2, s3=s3_client) @pytest.fixture diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py index f71dc3858e..6f7085a71a 100644 --- a/tests/test_datasets/test_dataset_functions.py +++ b/tests/test_datasets/test_dataset_functions.py @@ -22,7 +22,7 @@ import openml from openml import OpenMLDataset -from openml._api.clients.minio import MinIOClient +from openml._api.clients.s3 import S3Client from openml.datasets import edit_dataset, fork_dataset from openml.datasets.functions import ( DATASETS_CACHE_DIR_NAME, @@ -348,31 +348,31 @@ def test__getarff_path_dataset_arff(self): assert isinstance(arff_path, Path) assert arff_path.exists() - def test__download_minio_file_object_does_not_exist(self): - minio = MinIOClient() + def test__download_s3_file_object_does_not_exist(self): + s3_client = S3Client() self.assertRaisesRegex( # noqa: PT027 FileNotFoundError, r"Object at .* does not exist", - minio.download_minio_file, - source="http://data.openml.org/dataset20/i_do_not_exist.pq", + s3_client.download_file, + source="https://data.openml.org/dataset20/i_do_not_exist.pq", exists_ok=True, ) - def test__download_minio_file_works_with_bucket_subdirectory(self): - minio = MinIOClient() - file_destination = minio.download_minio_file( - source="http://data.openml.org/dataset61/dataset_61.pq", + def test__download_s3_file_works_with_bucket_subdirectory(self): + s3_client = S3Client() + file_destination = s3_client.download_file( + source="https://data.openml.org/dataset61/dataset_61.pq", exists_ok=True, ) assert os.path.isfile(file_destination), ( # noqa: PTH113 - "download_minio_file can download from subdirectories" + "download_file can download from subdirectories" ) - @mock.patch.object(MinIOClient, "download_minio_file") + @mock.patch.object(S3Client, "download_file") def test__get_dataset_parquet_is_cached(self, patch): with tempfile.TemporaryDirectory() as tmp_path: tmp_file_path = ( - Path(tmp_path) / "org" / "openml" / "data" / "minio" / "dataset30" / "dataset_30.pq" + Path(tmp_path) / "org" / "openml" / "data" / "s3" / "dataset30" / "dataset_30.pq" ) tmp_file_path.parent.mkdir(parents=True) with tmp_file_path.open(mode="w") as fh: @@ -384,7 +384,7 @@ def test__get_dataset_parquet_is_cached(self, patch): ) description = { - "oml:parquet_url": "http://data.openml.org/dataset30/dataset_30.pq", + "oml:parquet_url": "https://data.openml.org/dataset30/dataset_30.pq", "oml:id": "30", } path = _get_dataset_parquet(description) @@ -394,7 +394,7 @@ def test__get_dataset_parquet_is_cached(self, patch): def test__get_dataset_parquet_file_does_not_exist(self): description = { - "oml:parquet_url": "http://data.openml.org/dataset20/does_not_exist.pq", + "oml:parquet_url": "https://data.openml.org/dataset20/does_not_exist.pq", "oml:id": "20", } path = _get_dataset_parquet(description) @@ -1837,7 +1837,7 @@ def _dataset_features_is_downloaded(did: int): def _dataset_data_file_is_downloaded(dataset: OpenMLDataset): if dataset._parquet_url is not None: - pq_directory = Path(openml.config.get_minio_download_path(dataset._parquet_url)).parent + pq_directory = Path(openml.config.get_download_path(dataset._parquet_url)).parent if pq_directory.exists(): return any(f.suffix == ".pq" for f in pq_directory.iterdir()) arff_directory = ( @@ -1928,7 +1928,7 @@ def test_get_dataset_with_invalid_id() -> None: def test__get_dataset_parquet_not_cached(): description = { - "oml:parquet_url": "http://data.openml.org/dataset20/dataset_20.pq", + "oml:parquet_url": "https://data.openml.org/dataset20/dataset_20.pq", "oml:id": "20", } path = _get_dataset_parquet(description) diff --git a/tests/test_openml/test_api_calls.py b/tests/test_openml/test_api_calls.py index b22505047c..193b85e376 100644 --- a/tests/test_openml/test_api_calls.py +++ b/tests/test_openml/test_api_calls.py @@ -1,17 +1,12 @@ from __future__ import annotations import unittest.mock -from collections.abc import Iterable, Iterator -from pathlib import Path -from typing import NamedTuple -from unittest import mock -import minio import pytest import openml import openml.testing -from openml._api_calls import API_TOKEN_HELP_LINK, _download_minio_bucket +from openml._api_calls import API_TOKEN_HELP_LINK class TestConfig(openml.testing.TestBase): @@ -40,70 +35,6 @@ def test_retry_on_database_error(self, Session_class_mock, _): # noqa: PT019 assert Session_class_mock.return_value.__enter__.return_value.get.call_count == 20 -class FakeObject(NamedTuple): - object_name: str - etag: str - """We use the etag of a Minio object as the name of a marker if we already downloaded it.""" - - -class FakeMinio: - def __init__(self, objects: Iterable[FakeObject] | None = None): - self._objects = objects or [] - - def list_objects(self, *args, **kwargs) -> Iterator[FakeObject]: # noqa: ARG002 - yield from self._objects - - def fget_object(self, object_name: str, file_path: str, *args, **kwargs) -> None: # noqa: ARG002 - if object_name in [obj.object_name for obj in self._objects]: - Path(file_path).write_text("foo") - return - raise FileNotFoundError - - -@mock.patch.object(minio, "Minio") -def test_download_all_files_observes_cache(mock_minio, tmp_path: Path) -> None: - some_prefix, some_filename = "some/prefix", "dataset.arff" - some_object_path = f"{some_prefix}/{some_filename}" - some_url = f"https://not.real.com/bucket/{some_object_path}" - mock_minio.return_value = FakeMinio( - objects=[ - FakeObject(object_name=some_object_path, etag=str(hash(some_object_path))), - ], - ) - - _download_minio_bucket(source=some_url, destination=tmp_path) - time_created = (tmp_path / "dataset.arff").stat().st_ctime - - _download_minio_bucket(source=some_url, destination=tmp_path) - time_modified = (tmp_path / some_filename).stat().st_mtime - - assert time_created == time_modified - - -@mock.patch.object(minio, "Minio") -def test_download_minio_failure(mock_minio, tmp_path: Path) -> None: - some_prefix, some_filename = "some/prefix", "dataset.arff" - some_object_path = f"{some_prefix}/{some_filename}" - some_url = f"https://not.real.com/bucket/{some_object_path}" - mock_minio.return_value = FakeMinio( - objects=[ - FakeObject(object_name=None, etag="tmp"), - ], - ) - - with pytest.raises(ValueError): # noqa: PT011 - _download_minio_bucket(source=some_url, destination=tmp_path) - - mock_minio.return_value = FakeMinio( - objects=[ - FakeObject(object_name="tmp", etag=None), - ], - ) - - with pytest.raises(ValueError): # noqa: PT011 - _download_minio_bucket(source=some_url, destination=tmp_path) - - @pytest.mark.parametrize( ("endpoint", "method"), [ diff --git a/tests/test_tasks/test_task_functions.py b/tests/test_tasks/test_task_functions.py index 62be9f376a..1438a5f9f3 100644 --- a/tests/test_tasks/test_task_functions.py +++ b/tests/test_tasks/test_task_functions.py @@ -150,7 +150,7 @@ def test_get_task(self): dataset = openml.datasets.get_dataset(task.dataset_id) assert isinstance(task, OpenMLTask) assert os.path.exists( # noqa: PTH110 - openml.config.get_minio_download_path(dataset._parquet_url) + openml.config.get_download_path(dataset._parquet_url) ) @pytest.mark.test_server