# Copyright (C) 2018 Hans van Kranenburg <hans@knorrie.org>
#
# This file is part of the python-btrfs module.
#
# python-btrfs is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# python-btrfs is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with python-btrfs. If not, see <http://www.gnu.org/licenses/>.
"""
This module provides advanced usage reporting for a btrfs filesystem.
By calling the :func:`~btrfs.ctree.FileSystem.usage` function on a
:class:`btrfs.ctree.FileSystem` object, an :class:`FsUsage` object is returned
that can be inspected.
Example::
>>> import btrfs
>>> with btrfs.FileSystem('/') as fs:
... usage = fs.usage()
... btrfs.utils.pretty_print(usage)
.. note::
If you're not yet familiar with it, btrfs terminology can be quite
confusing.
Here's just an example: In btrfs terminology, a ‘space’ is the collection
of all block groups that have identical type and profile flags. For
example, Metadata, DUP is a ‘space’. The word ‘space’ is also used for the
distinction between ‘physical address space’ and ‘virtual address space’.
"""
import btrfs
import copy
from btrfs.ctree import ( # noqa
BLOCK_GROUP_DATA, BLOCK_GROUP_SYSTEM, BLOCK_GROUP_METADATA,
BLOCK_GROUP_TYPE_MASK, BLOCK_GROUP_PROFILE_MASK,
)
[docs]
class DevSpaceUsage(object):
"""Physical usage details for a single space per device.
For example, a `Data, DUP` chunk of 1GiB results in a 2GiB allocation of
physical bytes on the device. A `Data, RAID5` chunk of 3GiB, allocated over
4 devices results in a 1GiB allocation on each device, with 256MiB reserved
for parity.
:ivar int flags: Block group type and profile, e.g. `Data, RAID1`.
:ivar int devid: Device ID
:ivar int allocated: Total amount of allocated physical bytes.
:ivar int parity: Amount of allocated physical bytes reserved for parity.
.. note::
Objects of this type are provided as part of an :class:`FsUsage` object.
"""
def __init__(self, devid, flags):
self.flags = flags
self.devid = devid
self.allocated = 0
self.parity = 0
def _add_usage(self, allocated, parity):
self.allocated += allocated
self.parity += parity
@staticmethod
def _pretty_properties():
return [
(btrfs.utils.block_group_flags_str, 'flags'),
(btrfs.utils.pretty_size, 'allocated'),
(btrfs.utils.pretty_size, 'parity'),
]
[docs]
class DevUsage(object):
"""Physical usage details for a device.
:ivar int devid: Device ID
:ivar int total: Total amount of bytes.
:ivar int allocated: Total amount of allocated bytes.
:ivar dev_space_usage: Allocated and parity bytes per space for this
device, indexed by space flags.
:vartype dev_space_usage: dict of DevSpaceUsage
:ivar int unallocatable: Physical bytes that are not allocatable because of
unbalanced device sizes.
:ivar int unallocatable_reclaimable: Physical bytes that are not
allocatable because of unbalanced allocations.
.. note::
Objects of this type are provided as part of an :class:`FsUsage` object.
"""
def __init__(self, device):
self.devid = device.devid
self.total = device.total_bytes
self.allocated = device.bytes_used
self.unallocated = self.total - self.allocated
self.dev_space_usage = {}
self.unallocatable_soft = None # set during FsUsage init
self.unallocatable_hard = None # set during FsUsage init
self.unallocatable_reclaimable = None # set during FsUsage init
def _dev_space_usage_key_str(flags):
return btrfs.utils.block_group_flags_str(flags)
def _add_usage(self, flags, allocated, parity):
if flags not in self.dev_space_usage:
self.dev_space_usage[flags] = DevSpaceUsage(self.devid, flags)
self.dev_space_usage[flags]._add_usage(allocated, parity)
def _init_unallocatable_reclaimable(self):
self.unallocatable_reclaimable = \
max(self.unallocatable_soft - self.unallocatable_hard, 0)
@staticmethod
def _pretty_properties():
return [
(btrfs.utils.pretty_size, 'total'),
(btrfs.utils.pretty_size, 'allocated'),
(btrfs.utils.pretty_size, 'unallocated'),
(btrfs.utils.pretty_size, 'unallocatable_soft'),
(btrfs.utils.pretty_size, 'unallocatable_hard'),
(btrfs.utils.pretty_size, 'unallocatable_reclaimable'),
]
[docs]
class RawSpaceUsage(object):
"""Physical usage details per space.
For example, if the `Metadata, RAID1` space has a 2GiB size in terms of
virtual addressing, in which 768MiB is used, then the allocated physical
size is 4GiB and amount of physical bytes used is 1.5GiB. A `Data, RAID6`
space of 8GiB, consisting of two 4GiB block groups (virtual address space),
each distributed over 6 devices, will occupy 2*(4+2) = 12 GiB physical
allocated bytes and have 4GiB of allocated bytes reserved for parity
blocks.
:ivar int flags: Block group type and profile, e.g. `Data, RAID1`.
:ivar int allocated: Total amount of allocated bytes.
:ivar int parity: Total amount of allocated bytes reserved for parity
blocks.
:ivar int used: Total amount of physical bytes used.
.. note::
Objects of this type are provided as part of an :class:`FsUsage` object.
"""
def __init__(self, space):
self.flags = space.flags
ratio = btrfs.volumes.block_group_profile_ncopies(space.flags)
self.allocated = space.total_bytes * ratio # initially missing parity blocks
self.used = space.used_bytes * ratio
self.parity = 0
def _add_usage(self, parity):
self.allocated += parity
self.parity += parity
@staticmethod
def _pretty_properties():
return [
(btrfs.utils.block_group_flags_str, 'flags'),
(btrfs.utils.pretty_size, 'allocated'),
(btrfs.utils.pretty_size, 'parity'),
(btrfs.utils.pretty_size, 'used'),
]
[docs]
class BlockGroupTypeUsage(object):
"""Physical usage details per block group type.
Totals per block group type (`System`, `Data`, `Metadata`, or,
`Data+Metadata` for mixed mode), disregarding the block group profile
(`Single`, `RAID1`, etc).
:ivar int type: Block group type, e.g. `Metadata`.
:ivar int allocated: Total amount of allocated bytes.
:ivar int parity: Total amount of allocated bytes reserved for parity
blocks.
:ivar int used: Total amount of physical bytes used.
.. note::
Objects of this type are provided as part of an :class:`FsUsage` object.
"""
def __init__(self, block_group_type):
self.type = block_group_type
self.allocated = 0
self.used = 0
self.parity = 0
def _add_usage(self, allocated, parity, used):
self.allocated += allocated
self.parity += parity
self.used += used
@staticmethod
def _pretty_properties():
return [
(btrfs.utils.block_group_flags_str, 'type'),
(btrfs.utils.pretty_size, 'allocated'),
(btrfs.utils.pretty_size, 'parity'),
(btrfs.utils.pretty_size, 'used'),
]
[docs]
class VirtualSpaceUsage(object):
"""Virtual usage per space.
:ivar int flags: Block group type and profile, e.g. `Data, RAID1`.
:ivar int total: Total amount of allocated bytes for this space.
:ivar int used: Total amount of virtual bytes used.
.. note::
Objects of this type are provided as part of an :class:`FsUsage` object.
"""
def __init__(self, space):
self.flags = space.flags
self.total = space.total_bytes
self.used = space.used_bytes
@staticmethod
def _pretty_properties():
return [
(btrfs.utils.block_group_flags_str, 'flags'),
(btrfs.utils.pretty_size, 'total'),
(btrfs.utils.pretty_size, 'used'),
]
[docs]
class VirtualBlockGroupTypeUsage(object):
"""Virtual address space usage per block group type.
Totals for the virtual address space per block group type (`System`,
`Data`, `Metadata`, or, `Data+Metadata` for mixed mode), disregarding the
block group profile (`Single`, `RAID1`, etc).
:ivar int flags: Block group type, e.g. `Metadata`.
:ivar int total: Total amount of allocated bytes.
:ivar int used: Total amount of virtual bytes used.
:ivar int unused: Amount of allocated but unused virtual bytes.
.. note::
Objects of this type are provided as part of an :class:`FsUsage` object.
"""
def __init__(self, block_group_type):
self.type = block_group_type
self.total = 0
self.used = 0
self.unused = None
def _add_usage(self, total, used):
self.total += total
self.used += used
self.unused = self.total - self.used
@staticmethod
def _pretty_properties():
return [
(btrfs.utils.block_group_flags_str, 'type'),
(btrfs.utils.pretty_size, 'total'),
(btrfs.utils.pretty_size, 'used'),
(btrfs.utils.pretty_size, 'unused'),
]
[docs]
class FsUsage(object):
"""Detailed usage information for a file system.
When creating an object of this type, the first argument, fs, is mandatory.
The other arguments can be used to influence the simulation to predict free
space and unallocatable space with explicit hints instead of using
information from the current filesystem, This is used by the
space-calculator program to run the simulation starting with a completely
empty filesystem.
:param btrfs.ctree.FileSystem fs: Filesystem to examine.
:param int data_metadata_ratio: Data to metadata ratio to use when running
the simulation to predict free space and unallocatable space.
:param int target_profile_metadata: Explicitly set metadata profile to use
for new allocations when running the simulation to predict free
and unallocatable space (not for a mixed filesystem).
:param int target_profile_data: Explicitly set data profile to use for new
allocations when running the simulation to predict free and
unallocatable space (not for a mixed filesystem).
:param int target_profile_mixed: Explicitly set metadata and data profile
to use for new allocations when running the simulation to predict free
and unallocatable space (only for a mixed filesystem).
Target block group profiles (used for new chunk allocations):
:ivar int target_profile_system: Profile for new System chunk allocations.
:ivar int target_profile_metadata: Profile for new Metadata chunk
allocations (not for a mixed filesystem).
:ivar int target_profile_data: Profile for new Data chunk allocations (not
for a mixed filesystem).
:ivar int target_profile_mixed: Profile for new Data+Metadata chunk
allocations (only for a mixed filesystem).
Usage details for the physical address space:
:ivar int total: Total amount of physical bytes in the filesystem.
:ivar int allocated: Total amount of allocated physical bytes.
:ivar int parity: Total amount of allocated bytes reserved for parity
blocks.
:ivar dev_usage: Physical usage details per device, indexed by Device ID.
:vartype dev_usage: dict of DevUsage
:ivar block_group_type_usage: Physical usage details per block group type,
indexed by block group type.
:vartype block_group_type_usage: dict of BlockGroupTypeUsage
:ivar raw_space_usage: Physical usage details per space, indexed by space
flags.
:vartype raw_space_usage: dict of RawSpaceUsage
Usage details for the virtual address space:
:ivar int virtual_total: Total amount of virtual address space.
:ivar int virtual_used: Total amount of bytes used inside the virtual
address space.
:ivar virtual_block_group_type_usage: Virtual address space usage per block
group type, indexed by block group type.
:vartype virtual_block_group_type_usage: dict of VirtualBlockGroupTypeUsage
:ivar VirtualSpaceUsage virtual_space_usage: Virtual usage per space,
indexed by space flags.
:vartype virtual_space_usage: dict of VirtualSpaceUsage
Allocatable space information:
The *soft* unallocatable amount of bytes is the currently unallocatable
part of the physical bytes on attached devices because the allocations in
the filesystem are unbalanced. This value is estimated by extrapolating the
current usage pattern and simulating new chunk allocations using the
current target allocation profiles. By doing so, we also discover how much
extra *virtual* address space these allocations would result in.
:ivar int unallocatable_soft: Unallocatable physical disk space because of
unbalanced allocations.
:ivar int estimated_allocatable_virtual_metadata: Estimated amount of
virtual address space bytes that can be added by allocating physical
bytes for metadata, based on the current usage pattern (not for a mixed
filesystem).
:ivar int estimated_allocatable_virtual_data: Estimated amount of virtual
address space bytes that can be added by allocating physical bytes for
data, based on the current usage pattern (not for a mixed filesystem).
:ivar int estimated_allocatable_virtual_mixed: Estimated amount of virtual
address space bytes that can be added by allocating physical bytes for
metadata and data,, based on the current usage pattern (only for a
mixed filesystem).
The *hard* unallocatable amount of bytes is the amount of physical bytes
that cannot be used for allocations, because of having different sizes of
devices attached. These values are unallocatable disk space that remains
after trying to simulate data and metadata allocations in a ratio similar
to current usage, starting with all disks being empty.
:ivar int unallocatable_hard: Unallocatable physical disk space because of
unbalanced device sizes.
:ivar int estimated_full_allocatable_virtual_metadata: Estimated amount of
virtual address space bytes for metadata, in case of optimally balanced
allocations. (not for a mixed filesystem).
:ivar int estimated_full_allocatable_virtual_data: Estimated amount of
virtual address space bytes for data, in case of optimally balanced
allocations. (not for a mixed filesystem).
:ivar int estimated_full_allocatable_virtual_mixed: Estimated amount of
virtual address space bytes for metadata and data, in case of optimally
balanced allocations. (only for a mixed filesystem).
The difference between *soft* and *hard* unallocatable bytes is the amount
of physical disk space that can be reclaimed for allocations when
rebalancing the filesystem.
:ivar int unallocatable_reclaimable: Unallocatable physical bytes that can
be reclaimed when balancing the filesystem.
Some other totals for convenience:
:ivar int allocatable: The total amount of physical bytes that are
allocatable in this filesystem. I.e. total size minus
unallocatable_soft.
:ivar int allocatable_left: The amount of allocatable physical bytes
remaining, until the filesystem will report being out of space, given
current usage pattern and target profiles.
By combining the unused virtual space in already allocated chunks and
estimated allocatable virtual bytes, we get actual numbers of estimated
free space. I.e., what we would like df to show us.
:ivar int free_metadata: Estimated virtual space left to use for metadata
(not for a mixed filesystem).
:ivar int free_data: Estimated virtual space left to use for data (not for
a mixed filesystem).
:ivar int free_mixed: Estimated virtual space left to use for metadata and
data (only for a mixed filesystem).
"""
def __init__(self, fs, data_metadata_ratio=None,
target_profile_metadata=None,
target_profile_data=None,
target_profile_mixed=None):
self._mixed_groups = fs.mixed_groups()
# Spaces and devices are a source of information
spaces = [
space
for space in fs.space_info()
if space.flags != btrfs.ctree.SPACE_INFO_GLOBAL_RSV
]
devices = list(fs.devices())
self.raw_space_usage = {
space.flags: RawSpaceUsage(space)
for space in spaces
}
self.dev_usage = {
device.devid: DevUsage(device)
for device in devices
}
self.total = sum(
device.total_bytes
for device in devices
)
self.allocated = sum(
device.bytes_used
for device in devices
)
self.parity = 0
BLOCK_GROUP_MIXED = btrfs.BLOCK_GROUP_METADATA | btrfs.BLOCK_GROUP_DATA
if not self._mixed_groups:
if target_profile_metadata is not None:
self.target_profile_system = target_profile_metadata | btrfs.BLOCK_GROUP_SYSTEM
self.target_profile_metadata = target_profile_metadata | btrfs.BLOCK_GROUP_METADATA
if target_profile_data is not None:
self.target_profile_data = target_profile_data | btrfs.BLOCK_GROUP_DATA
else:
if target_profile_mixed is not None:
self.target_profile_system = target_profile_mixed | btrfs.BLOCK_GROUP_SYSTEM
self.target_profile_mixed = target_profile_mixed | BLOCK_GROUP_MIXED
# We walk the chunk list because every block group / chunk can be laid
# out over any amount of disks. To collect e.g. the amount of parity
# bytes, we need to look at all of them.
#
# Confusing: chunk.type is actually all the flags, so type and profile
# combined.
for chunk in fs.chunks():
flags = chunk.type
if flags not in self.raw_space_usage:
continue # A conversion to this profile just started right now?
# Remember last seen chunk types as target profiles
block_group_type = flags & BLOCK_GROUP_TYPE_MASK
if block_group_type == BLOCK_GROUP_SYSTEM:
self.target_profile_system = flags
elif not self._mixed_groups:
if block_group_type == BLOCK_GROUP_DATA:
self.target_profile_data = flags
elif block_group_type == BLOCK_GROUP_METADATA:
self.target_profile_metadata = flags
else:
self.target_profile_mixed = flags
dev_extent_length = btrfs.volumes.chunk_to_dev_extent_length(chunk)
chunk_raw_parity_bytes = btrfs.volumes.chunk_to_raw_parity_bytes(chunk)
self.parity += chunk_raw_parity_bytes
self.raw_space_usage[flags]._add_usage(chunk_raw_parity_bytes)
dev_extent_parity_bytes = chunk_raw_parity_bytes // len(chunk.stripes)
for stripe in chunk.stripes:
if stripe.devid not in self.dev_usage:
continue # A device just got added?
self.dev_usage[stripe.devid]._add_usage(flags,
dev_extent_length,
dev_extent_parity_bytes)
# Combine information from different spaces with same chunk type into
# totals per block group type. So, e.g. all DATA space, regardless of
# being single, RAID1, etc...
self.block_group_type_usage = {}
for raw_space in self.raw_space_usage.values():
space_type = raw_space.flags & BLOCK_GROUP_TYPE_MASK
if space_type not in self.block_group_type_usage:
self.block_group_type_usage[space_type] = BlockGroupTypeUsage(space_type)
self.block_group_type_usage[space_type]._add_usage(
raw_space.allocated, raw_space.parity, raw_space.used)
self.virtual_space_usage = {
space.flags: VirtualSpaceUsage(space)
for space in spaces
}
# Combine information from different spaces with same chunk type into
# totals per block group type. So, e.g. all DATA space, regardless of
# being single, RAID1, etc...
self.virtual_block_group_type_usage = {}
for virtual_space in self.virtual_space_usage.values():
space_type = virtual_space.flags & BLOCK_GROUP_TYPE_MASK
if space_type not in self.virtual_block_group_type_usage:
self.virtual_block_group_type_usage[space_type] = \
VirtualBlockGroupTypeUsage(space_type)
self.virtual_block_group_type_usage[space_type]._add_usage(
virtual_space.total, virtual_space.used)
# The total size of the block groups in the virtual address space
self.virtual_total = sum(
virtual_space.total
for virtual_space in self.virtual_space_usage.values()
)
self.virtual_used = sum(
virtual_space.used
for virtual_space in self.virtual_space_usage.values()
)
if data_metadata_ratio is not None:
self.default_data_metadata_ratio = data_metadata_ratio
else:
self.default_data_metadata_ratio = 200
# Estimate the amount of unallocatable raw disk space if the sizes of
# attached block devices are unbalanced. We start the simulation with
# the entire sizes of the attached devices and keep allocating chunks
# until not possible any more.
#
device_sizes = {
devid: dev_usage.total
for devid, dev_usage in self.dev_usage.items()
}
if not self._mixed_groups:
# The estimated "full" allocatable numbers are the estimation of
# virtual space to be used for data and metadata when the
# filesystem would be totally empty to begin with.
#
# This is e.g. used by the space calculator example.
dev_unallocatable_hard, \
self.estimated_full_allocatable_virtual_metadata, \
self.estimated_full_allocatable_virtual_data = \
self._simulate_chunk_allocations(device_sizes)
else:
dev_unallocatable_hard, \
self.estimated_full_allocatable_virtual_mixed = \
self._simulate_chunk_allocations(device_sizes)
for devid, unallocatable_hard in dev_unallocatable_hard.items():
self.dev_usage[devid].unallocatable_hard = unallocatable_hard
self.unallocatable_hard = sum(dev_unallocatable_hard.values())
# Next, we estimate the amount of unallocatable raw disk space when
# starting out with the current state of the filesystem.
unallocated_sizes = {
devid: dev_usage.unallocated
for devid, dev_usage in self.dev_usage.items()
}
if not self._mixed_groups:
dev_unallocatable_soft, \
self.estimated_allocatable_virtual_metadata, \
self.estimated_allocatable_virtual_data = \
self._simulate_chunk_allocations(unallocated_sizes)
else:
dev_unallocatable_soft, \
self.estimated_allocatable_virtual_mixed = \
self._simulate_chunk_allocations(unallocated_sizes)
for devid, unallocatable_soft in dev_unallocatable_soft.items():
self.dev_usage[devid].unallocatable_soft = unallocatable_soft
self.unallocatable_soft = sum(dev_unallocatable_soft.values())
# At this point, it is possible that the unallocatable_hard amounts are
# higher than the unallocatable_soft amounts. E.g. if we just switched
# to another target profile. In that case, this means we can not fully
# rewrite all data to the target profile!
# For convenience reasons, we provide a few more derived numbers...
#
# If the soft unallocatable amount of bytes is higher than the hard
# amount, we can reclaim space by balancing the filesytsem.
self.unallocatable_reclaimable = \
max(self.unallocatable_soft - self.unallocatable_hard, 0)
for dev_usage in self.dev_usage.values():
dev_usage._init_unallocatable_reclaimable()
self.allocatable = self.total - self.unallocatable_soft
self.allocatable_left = self.allocatable - self.allocated
if not self._mixed_groups:
self.free_metadata = self.estimated_allocatable_virtual_metadata
if btrfs.BLOCK_GROUP_METADATA in self.virtual_block_group_type_usage:
self.free_metadata += \
self.virtual_block_group_type_usage[btrfs.BLOCK_GROUP_METADATA].unused
self.free_data = self.estimated_allocatable_virtual_data
if btrfs.BLOCK_GROUP_DATA in self.virtual_block_group_type_usage:
self.free_data += \
self.virtual_block_group_type_usage[btrfs.BLOCK_GROUP_DATA].unused
else:
self.free_mixed = self.estimated_allocatable_virtual_mixed
if BLOCK_GROUP_MIXED in self.virtual_block_group_type_usage:
self.free_mixed += \
self.virtual_block_group_type_usage[BLOCK_GROUP_MIXED].unused
def _raw_space_usage_key_str(flags):
return btrfs.utils.block_group_flags_str(flags)
def _block_group_type_usage_key_str(block_group_type):
return btrfs.utils.block_group_flags_str(block_group_type)
def _virtual_space_usage_key_str(flags):
return btrfs.utils.block_group_flags_str(flags)
def _virtual_block_group_type_usage_key_str(block_group_type):
return btrfs.utils.block_group_flags_str(block_group_type)
def _data_metadata_ratio(self):
"""
Determine data to metadata ratio for allocation simulation for wasted space.
E.g. a ratio of 200 means that for every X bytes of metadata, we can
allocate 200*X bytes for data.
Actual simulation ratio is a weighted combination of the current data to
metadata ratio and the default of 200, where the weight of the current
ratio increases if the filesystem is filled up more.
If an empty filesystem is presented (this is used for the space
calculator), then just use the default that was set earlier.
"""
if self._mixed_groups:
raise ValueError("Data to metadata ratio is irrelevant for mixed groups.")
if BLOCK_GROUP_METADATA not in self.virtual_block_group_type_usage:
return self.default_data_metadata_ratio
used_fraction = self.virtual_used / self.total
used_metadata = self.virtual_block_group_type_usage[BLOCK_GROUP_METADATA].used
used_data = self.virtual_block_group_type_usage[BLOCK_GROUP_DATA].used
used_ratio = used_data / used_metadata
return used_fraction * used_ratio + (1 - used_fraction) * self.default_data_metadata_ratio
def _alloc_chunk(self, sizes, flags):
"""
This is used by the wasted space calculator.
sizes is a dictionary {devid: allocatable_bytes, ...}
which will be modified in place as side effect
flags contains allocation type, which is DATA (also for mixed) or METADATA
This function tries to reduce unallocated raw bytes on each disk in a way
similar to the workings of the btrfs chunk allocator. The sizes dictionary
will be modified in place while doing so. When returning False, no chunk
allocation is possible any more, and sizes will show the amount of
unallocatable bytes per device.
"""
attrs = btrfs.volumes._raid_attrs(flags & BLOCK_GROUP_PROFILE_MASK)
if flags & BLOCK_GROUP_DATA:
max_stripe_size = btrfs.utils.SZ_1G
max_chunk_size = btrfs.volumes.BTRFS_MAX_DATA_CHUNK_SIZE
elif flags & BLOCK_GROUP_METADATA:
if self.total > 50 * btrfs.utils.SZ_1G:
max_stripe_size = btrfs.utils.SZ_1G
else:
max_stripe_size = btrfs.utils.SZ_256M
max_chunk_size = max_stripe_size
else:
raise ValueError("Only DATA and METADATA supported here")
# we don't want a chunk larger than 10% of writeable space
max_chunk_size = min(self.total // 10, max_chunk_size)
# [(devid, unallocated), ...], most unallocated space per device first
non_zero_sizes = {
devid: unallocated
for devid, unallocated in sizes.items()
if unallocated > 0
}
sorted_sizes = sorted(list(non_zero_sizes.items()), key=lambda x: -x[1])
# Keep a multiple of devs_increment, chop off the rest
sorted_sizes = sorted_sizes[:len(sorted_sizes) -
(len(sorted_sizes) % attrs.devs_increment)]
if len(sorted_sizes) < attrs.devs_min:
return 0
# Keep only the amount we need for a single chunk allocation
if attrs.devs_max != 0:
sorted_sizes = sorted_sizes[:min(len(sorted_sizes), attrs.devs_max)]
# Actual device extent size is limited by the device with least amount of
# available space and by max_stripe_size.
stripe_size = min(max_stripe_size, sorted_sizes[-1][1] // attrs.dev_stripes)
# But, there's another limit, the max_chunk_size...
num_stripes = len(sorted_sizes) * attrs.dev_stripes
chunk_size = btrfs.volumes.dev_extent_length_to_chunk_length(
flags, num_stripes, stripe_size)
if chunk_size > max_chunk_size:
stripe_size = btrfs.volumes.chunk_length_to_dev_extent_length(
flags, num_stripes, max_chunk_size)
chunk_size = max_chunk_size
# Finally, decrease unallocated space
for devid, _ in sorted_sizes:
sizes[devid] -= stripe_size * attrs.dev_stripes
return chunk_size
def _simulate_chunk_allocations(self, sizes):
"""
Try to do metadata and data allocations until no longer possible. The sizes
dictionary is modified in place and lists the amount of unallocatable space
per disk when the function returns.
"""
_sizes = copy.deepcopy(sizes) # copy will be modified in place
if not self._mixed_groups:
ratio = self._data_metadata_ratio()
metadata_flags = self.target_profile_metadata
data_flags = self.target_profile_data
virtual_data = 0
virtual_metadata = 0
while True:
chunk_size = self._alloc_chunk(_sizes, metadata_flags)
if chunk_size == 0:
return _sizes, virtual_metadata, virtual_data
virtual_metadata += chunk_size
while virtual_data / virtual_metadata < ratio:
chunk_size = self._alloc_chunk(_sizes, data_flags)
if chunk_size == 0:
return _sizes, virtual_metadata, virtual_data
virtual_data += chunk_size
else:
flags = self.target_profile_mixed
virtual_mixed = 0
while True:
chunk_size = self._alloc_chunk(_sizes, flags)
if chunk_size == 0:
return _sizes, virtual_mixed
virtual_mixed += chunk_size
@staticmethod
def _pretty_properties():
return [
(btrfs.utils.space_profile_description, 'target_profile_system'),
(btrfs.utils.space_profile_description, 'target_profile_metadata'),
(btrfs.utils.space_profile_description, 'target_profile_data'),
(btrfs.utils.space_profile_description, 'target_profile_mixed'),
(btrfs.utils.btrfs.utils.pretty_size, 'total'),
(btrfs.utils.btrfs.utils.pretty_size, 'allocated'),
(btrfs.utils.btrfs.utils.pretty_size, 'parity'),
(btrfs.utils.btrfs.utils.pretty_size, 'virtual_total'),
(btrfs.utils.btrfs.utils.pretty_size, 'virtual_used'),
(btrfs.utils.btrfs.utils.pretty_size, 'unalloctable_soft'),
(btrfs.utils.btrfs.utils.pretty_size, 'estimated_allocatable_virtual_metadata'),
(btrfs.utils.btrfs.utils.pretty_size, 'estimated_allocatable_virtual_data'),
(btrfs.utils.btrfs.utils.pretty_size, 'estimated_allocatable_virtual_mixed'),
(btrfs.utils.btrfs.utils.pretty_size, 'unallocatable_hard'),
(btrfs.utils.btrfs.utils.pretty_size, 'estimated_full_allocatable_virtual_metadata'),
(btrfs.utils.btrfs.utils.pretty_size, 'estimated_full_allocatable_virtual_data'),
(btrfs.utils.btrfs.utils.pretty_size, 'estimated_full_allocatable_virtual_mixed'),
(btrfs.utils.btrfs.utils.pretty_size, 'unallocatable_reclaimable'),
(btrfs.utils.btrfs.utils.pretty_size, 'allocatable'),
(btrfs.utils.btrfs.utils.pretty_size, 'allocatable_left'),
(btrfs.utils.btrfs.utils.pretty_size, 'free_metadata'),
(btrfs.utils.btrfs.utils.pretty_size, 'free_data'),
(btrfs.utils.btrfs.utils.pretty_size, 'free_mixed'),
]