📤 copy_to_destination#

Copies the built datapack, libraries and resource pack to every configured destination.

CACHE_NAME: str = 'stewbeet_copy_destinations'[source]#

Beet cache slot holding the sha1 of every file already sent to each remote destination.

MAX_COPY_WORKERS: int = 8[source]#

Upper bound on concurrent copies: SFTP puts are latency bound, local copies are disk bound.

class CopyTask(src: str, dst: str, group: str)[source]#

Bases: object

A single file to place at a single destination.

src: str[source]#

Absolute or relative path of the local file to copy.

dst: str[source]#

Destination, either a local path or an sftp://user@host/path URL.

group: str[source]#

Label used to group destinations in the summary log.

class CopyReport(copied: set[str] = <factory>, skipped: int = 0)[source]#

Bases: object

Outcome of a batch of copy tasks.

copied: set[str][source]#

Groups for which at least one file was actually written.

skipped: int = 0[source]#

Number of remote files left untouched because they were already up to date.

beet_default(ctx: Context) → None[source]#

Copy destination plugin for StewBeet. Copies the datapack (not merged) and all libs zips to all datapack destinations. Copies the resource pack (merged if available, otherwise normal) to all resource pack destinations.

Parameters:

ctx – The beet context.

sftp#

Pooled SFTP connections shared by every plugin that writes to an sftp:// destination.

Opening an SSH connection costs roughly half a second: importing paramiko, the key exchange, the authentication and the first channel. That dwarfs the transfers themselves when a single zip changed. One authenticated transport is therefore kept per server for the whole process, and each thread opens a cheap channel on top of it instead of shaking hands again. SftpPool.warmup_from_context starts all of that in the background at the very beginning of the build, so the connection is already waiting by the time the copy plugin runs at the end.

CREDENTIALS_PATH: str = '~/stewbeet/credentials.yml'[source]#

YAML file holding the sftp: {"user@host": {password: ...}} entries used by destinations without an inline password.

DESTINATION_KEYS: tuple[str, ...] = ('datapack', 'resource_pack')[source]#

Keys of meta.stewbeet.build_copy_destinations that can hold remote destinations.

TIMEOUT_SECONDS: float = 30.0[source]#

Longest wait on the server for any single step: TCP connect, banner, authentication, channel opening or one SFTP request.

KEEPALIVE_SECONDS: int = 15[source]#

Interval of the keepalive packets keeping the transport open between the warm-up and the copy at the end of the build.

is_sftp_path(path: str) → bool[source]#

Return whether the given destination is a remote SFTP URL rather than a local path.

Parameters:

path – The destination to test.

Returns:

True when the destination is an sftp:// URL.

Return type:

bool

>>> is_sftp_path("sftp://bob@example.com/datapacks")
True
>>> is_sftp_path("D:/minecraft/latest/resourcepacks")
False
remote_path_of(url: str) → str[source]#

Return the server-side path of a destination URL.

Parameters:

url – An sftp://user[:pass]@host[:port]/path URL.

Returns:

The path part of the URL, always POSIX style.

Return type:

str

>>> remote_path_of("sftp://bob@example.com/sftp/Switch/datapacks/x.zip")
'/sftp/Switch/datapacks/x.zip'
class SftpEndpoint(
host: str,
port: int,
username: str,
password: str | None,
)[source]#

Bases: object

The SSH server behind one or more sftp:// destination URLs.

host: str[source]#

Hostname of the server.

port: int[source]#

SSH port, 22 unless the URL says otherwise.

username: str[source]#

User to authenticate as.

password: str | None[source]#

Password to authenticate with, or None to let paramiko try the local keys and agent.

static resolve_password(netloc: str) → str | None[source]#

Read the password of a user@host pair from the stewbeet credentials file.

Cached for the whole build: every destination resolves its endpoint, and re-parsing the credentials file each time costs more than the upload it is preparing.

Parameters:

netloc – The user@host part of the destination URL, used as credentials key.

Returns:

The configured password, or None when there is none to be found.

Return type:

str | None

static from_url(
url: str,
) → SftpEndpoint[source]#

Parse a destination URL into the server it points at.

Parameters:

url – An sftp://user[:pass]@host[:port]/path URL.

Returns:

The server behind that URL, password taken from the credentials file when the URL has none.

Return type:

SftpEndpoint

>>> SftpEndpoint.from_url("sftp://bob:hunter2@example.com/datapacks/x.zip")
SftpEndpoint(host='example.com', port=22, username='bob', password='hunter2')
class SftpChannels[source]#

Bases: _local

Per-thread SFTP channel holder: a channel is cheap to open but not safe to share between threads.

channel: SFTPClient | None = None[source]#

This thread’s channel on the shared transport, opened on first use.

class SftpConnection(
endpoint: SftpEndpoint,
ready: Event = <factory>,
client: SSHClient | None = None,
error: BaseException | None = None,
channels: SftpChannels = <factory>,
)[source]#

Bases: object

One authenticated SSH transport, plus one SFTP channel per thread that asks for it.

endpoint: SftpEndpoint[source]#

The server this connection talks to.

ready: threading.Event[source]#

Set once the connection attempt finished, successfully or not.

client: SSHClient | None = None[source]#

The authenticated client, or None while connecting and after a failure.

error: BaseException | None = None[source]#

The failure raised while connecting, kept to be re-raised in the threads asking for a channel.

channels: SftpChannels[source]#

The per-thread channels opened on this connection.

connect() → None[source]#

Open and authenticate the transport, storing any failure instead of raising it.

This usually runs in the warm-up thread, where nothing would catch an exception. The error is kept and re-raised by channel, so a broken connection is reported where the copy actually happens.

is_usable() → bool[source]#

Return whether this connection is still worth reusing, either still connecting or holding a live transport.

Returns:

False once the handshake failed or the server dropped the transport between two builds.

Return type:

bool

channel() → SFTPClient[source]#

Return this thread’s SFTP channel, opening one on the shared transport when needed.

Waits for the background warm-up to be done. Opening a channel is a single round trip, unlike the full handshake connect already paid once for the whole build.

Returns:

A channel this thread can use on its own.

Return type:

SFTPClient

class SftpPool[source]#

Bases: object

Process-wide registry of SFTP connections, one per server.

CONNECTIONS: ClassVar[dict[SftpEndpoint, SftpConnection]] = {}[source]#

Live connections, keyed by server so several destinations on the same host share one transport.

LOCK: ClassVar[lock] = <unlocked _thread.lock object>[source]#

Guards CONNECTIONS so the warm-up and the copy workers cannot open two transports for one server.

static get(
endpoint: SftpEndpoint,
) → SftpConnection[source]#

Return the connection to a server, opening it now when nothing opened it yet.

Parameters:

endpoint – The server to reach.

Returns:

The shared connection, possibly still being opened by another thread.

Return type:

SftpConnection

static channel_for(url: str) → tuple[SFTPClient, str][source]#

Return the channel serving a destination URL, along with the path to write to on the server.

Parameters:

url – An sftp://user[:pass]@host[:port]/path URL.

Returns:

This thread’s channel, and the remote path parsed out of the URL.

Return type:

tuple[SFTPClient, str]

static warmup(
urls: Iterable[str],
) → None[source]#

Start connecting to every server behind the given destinations, in the background.

Called at the beginning of the build so the handshake overlaps the rest of the pipeline: by the time the copy plugin runs at the end, get finds a connection that is already open. Connections left dead by a previous build are dropped first, which is what makes stewbeet watch reconnect on its own.

Parameters:

urls – Destinations to warm up, local paths are ignored.

static warmup_from_context(
ctx: Context,
) → None[source]#

Start warming up every remote destination listed in meta.stewbeet.build_copy_destinations.

Parameters:

ctx – The beet context.

static list_sizes(url: str) → dict[str, int] | None[source]#

List the directory holding a destination, as a mapping of remote path to file size.

Listing costs one round trip and keeps the upload cache honest: a file deleted or truncated server-side gets uploaded again instead of being wrongly considered up to date. A missing directory is detected from the listing itself, which saves the extra round trip an existence check would cost.

Parameters:

url – An sftp://user[:pass]@host[:port]/path URL pointing at a file.

Returns:

Sizes of every entry of the parent directory, or None if it does not exist.

Return type:

dict[str, int] | None

static put(url: str, src: str) → None[source]#

Upload a local file to a destination URL.

Parameters:
  • url – The sftp:// destination to write to.

  • src – Path of the local file to send.

static remove(url: str) → None[source]#

Delete the file a destination URL points at, doing nothing when it is already gone.

Parameters:

url – The sftp:// destination to delete.

silent#