diff options
author | Mark Salyzyn | 2018-11-07 09:40:31 -0600 |
---|---|---|
committer | Mark Salyzyn | 2018-11-08 17:42:22 -0600 |
commit | 3cd2460edc190d16be46388f2d39cac59f2ac182 (patch) | |
tree | e62a200a627eabca9d784a877bac0c7d040eaec8 /tools/releasetools | |
parent | c21c709f0b2929bc3e52c890d975572ac9a9e84e (diff) | |
download | platform-build-3cd2460edc190d16be46388f2d39cac59f2ac182.tar.gz platform-build-3cd2460edc190d16be46388f2d39cac59f2ac182.tar.xz platform-build-3cd2460edc190d16be46388f2d39cac59f2ac182.zip |
build: Split out mkfs in BuildImageMkfs (retry)
NB: second attempt, issue with scope for mkfs_output.
Separate out BuildImageMkfs from BuildImage, which just makes the
filesystem without any verity, avb, or other decisions. BuildImage
does all the wrapping for such. This will hopefully ease maintenance
and drop the issues surrounding BuildImage reentrancy.
Change right-size estimation path to use BuildImageMkfs, and thus do
so without verity or avb wrappings. Add partition_headroom to space
consideration. This makes the results of the estimation more
accurate and predictable.
Test: build
Bug: 111302946
Change-Id: I2549bd4e403c21290470b2fa1835492ae883f0fd
Diffstat (limited to 'tools/releasetools')
-rwxr-xr-x | tools/releasetools/build_image.py | 191 |
1 files changed, 110 insertions, 81 deletions
diff --git a/tools/releasetools/build_image.py b/tools/releasetools/build_image.py index 7611a4d43..d32090a98 100755 --- a/tools/releasetools/build_image.py +++ b/tools/releasetools/build_image.py | |||
@@ -221,8 +221,8 @@ def CheckHeadroom(ext4fs_output, prop_dict): | |||
221 | adjusted_blocks)) | 221 | adjusted_blocks)) |
222 | 222 | ||
223 | 223 | ||
224 | def BuildImage(in_dir, prop_dict, out_file, target_out=None): | 224 | def BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config): |
225 | """Builds an image for the files under in_dir and writes it to out_file. | 225 | """Builds a pure image for the files under in_dir and writes it to out_file. |
226 | 226 | ||
227 | Args: | 227 | Args: |
228 | in_dir: Path to input directory. | 228 | in_dir: Path to input directory. |
@@ -233,81 +233,15 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): | |||
233 | points to the /system directory under PRODUCT_OUT. fs_config (the one | 233 | points to the /system directory under PRODUCT_OUT. fs_config (the one |
234 | under system/core/libcutils) reads device specific FS config files from | 234 | under system/core/libcutils) reads device specific FS config files from |
235 | there. | 235 | there. |
236 | fs_config: The fs_config file that drives the prototype | ||
236 | 237 | ||
237 | Raises: | 238 | Raises: |
238 | BuildImageError: On build image failures. | 239 | BuildImageError: On build image failures. |
239 | """ | 240 | """ |
240 | original_mount_point = prop_dict["mount_point"] | ||
241 | in_dir, fs_config = SetUpInDirAndFsConfig(in_dir, prop_dict) | ||
242 | |||
243 | build_command = [] | 241 | build_command = [] |
244 | fs_type = prop_dict.get("fs_type", "") | 242 | fs_type = prop_dict.get("fs_type", "") |
245 | run_e2fsck = False | 243 | run_e2fsck = False |
246 | 244 | ||
247 | fs_spans_partition = True | ||
248 | if fs_type.startswith("squash"): | ||
249 | fs_spans_partition = False | ||
250 | |||
251 | # Get a builder for creating an image that's to be verified by Verified Boot, | ||
252 | # or None if not applicable. | ||
253 | verity_image_builder = verity_utils.CreateVerityImageBuilder(prop_dict) | ||
254 | |||
255 | if (prop_dict.get("use_dynamic_partition_size") == "true" and | ||
256 | "partition_size" not in prop_dict): | ||
257 | # If partition_size is not defined, use output of `du' + reserved_size. | ||
258 | size = GetDiskUsage(in_dir) | ||
259 | logger.info( | ||
260 | "The tree size of %s is %d MB.", in_dir, size // BYTES_IN_MB) | ||
261 | # If not specified, give us 16MB margin for GetDiskUsage error ... | ||
262 | size += int(prop_dict.get("partition_reserved_size", BYTES_IN_MB * 16)) | ||
263 | # Round this up to a multiple of 4K so that avbtool works | ||
264 | size = common.RoundUpTo4K(size) | ||
265 | if fs_type.startswith("ext"): | ||
266 | if verity_image_builder: | ||
267 | size = verity_image_builder.CalculateDynamicPartitionSize(size) | ||
268 | prop_dict["partition_size"] = str(size) | ||
269 | if "extfs_inode_count" not in prop_dict: | ||
270 | prop_dict["extfs_inode_count"] = str(GetInodeUsage(in_dir)) | ||
271 | logger.info( | ||
272 | "First Pass based on estimates of %d MB and %s inodes.", | ||
273 | size // BYTES_IN_MB, prop_dict["extfs_inode_count"]) | ||
274 | prop_dict["mount_point"] = original_mount_point | ||
275 | BuildImage(in_dir, prop_dict, out_file, target_out) | ||
276 | fs_dict = GetFilesystemCharacteristics(out_file) | ||
277 | os.remove(out_file) | ||
278 | block_size = int(fs_dict.get("Block size", "4096")) | ||
279 | free_size = int(fs_dict.get("Free blocks", "0")) * block_size | ||
280 | reserved_size = int(prop_dict.get("partition_reserved_size", 0)) | ||
281 | if free_size <= reserved_size: | ||
282 | logger.info( | ||
283 | "Not worth reducing image %d <= %d.", free_size, reserved_size) | ||
284 | else: | ||
285 | size -= free_size | ||
286 | size += reserved_size | ||
287 | if block_size <= 4096: | ||
288 | size = common.RoundUpTo4K(size) | ||
289 | else: | ||
290 | size = ((size + block_size - 1) // block_size) * block_size | ||
291 | extfs_inode_count = prop_dict["extfs_inode_count"] | ||
292 | inodes = int(fs_dict.get("Inode count", extfs_inode_count)) | ||
293 | inodes -= int(fs_dict.get("Free inodes", "0")) | ||
294 | prop_dict["extfs_inode_count"] = str(inodes) | ||
295 | prop_dict["partition_size"] = str(size) | ||
296 | logger.info( | ||
297 | "Allocating %d Inodes for %s.", inodes, out_file) | ||
298 | if verity_image_builder: | ||
299 | size = verity_image_builder.CalculateDynamicPartitionSize(size) | ||
300 | prop_dict["partition_size"] = str(size) | ||
301 | logger.info( | ||
302 | "Allocating %d MB for %s.", size // BYTES_IN_MB, out_file) | ||
303 | |||
304 | prop_dict["image_size"] = prop_dict["partition_size"] | ||
305 | |||
306 | # Adjust the image size to make room for the hashes if this is to be verified. | ||
307 | if verity_image_builder: | ||
308 | max_image_size = verity_image_builder.CalculateMaxImageSize() | ||
309 | prop_dict["image_size"] = str(max_image_size) | ||
310 | |||
311 | if fs_type.startswith("ext"): | 245 | if fs_type.startswith("ext"): |
312 | build_command = [prop_dict["ext_mkuserimg"]] | 246 | build_command = [prop_dict["ext_mkuserimg"]] |
313 | if "extfs_sparse_flag" in prop_dict: | 247 | if "extfs_sparse_flag" in prop_dict: |
@@ -400,8 +334,8 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): | |||
400 | logger.exception("Failed to compute disk usage with du") | 334 | logger.exception("Failed to compute disk usage with du") |
401 | du_str = "unknown" | 335 | du_str = "unknown" |
402 | print( | 336 | print( |
403 | "Out of space? The tree size of {} is {}, with reserved space of {} " | 337 | "Out of space? Out of inodes? The tree size of {} is {}, " |
404 | "bytes ({} MB).".format( | 338 | "with reserved space of {} bytes ({} MB).".format( |
405 | in_dir, du_str, | 339 | in_dir, du_str, |
406 | int(prop_dict.get("partition_reserved_size", 0)), | 340 | int(prop_dict.get("partition_reserved_size", 0)), |
407 | int(prop_dict.get("partition_reserved_size", 0)) // BYTES_IN_MB)) | 341 | int(prop_dict.get("partition_reserved_size", 0)) // BYTES_IN_MB)) |
@@ -414,6 +348,111 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): | |||
414 | int(prop_dict["partition_size"]) // BYTES_IN_MB)) | 348 | int(prop_dict["partition_size"]) // BYTES_IN_MB)) |
415 | raise | 349 | raise |
416 | 350 | ||
351 | if run_e2fsck and prop_dict.get("skip_fsck") != "true": | ||
352 | unsparse_image = UnsparseImage(out_file, replace=False) | ||
353 | |||
354 | # Run e2fsck on the inflated image file | ||
355 | e2fsck_command = ["e2fsck", "-f", "-n", unsparse_image] | ||
356 | try: | ||
357 | common.RunAndCheckOutput(e2fsck_command) | ||
358 | finally: | ||
359 | os.remove(unsparse_image) | ||
360 | |||
361 | return mkfs_output | ||
362 | |||
363 | |||
364 | def BuildImage(in_dir, prop_dict, out_file, target_out=None): | ||
365 | """Builds an image for the files under in_dir and writes it to out_file. | ||
366 | |||
367 | Args: | ||
368 | in_dir: Path to input directory. | ||
369 | prop_dict: A property dict that contains info like partition size. Values | ||
370 | will be updated with computed values. | ||
371 | out_file: The output image file. | ||
372 | target_out: Path to the TARGET_OUT directory as in Makefile. It actually | ||
373 | points to the /system directory under PRODUCT_OUT. fs_config (the one | ||
374 | under system/core/libcutils) reads device specific FS config files from | ||
375 | there. | ||
376 | |||
377 | Raises: | ||
378 | BuildImageError: On build image failures. | ||
379 | """ | ||
380 | in_dir, fs_config = SetUpInDirAndFsConfig(in_dir, prop_dict) | ||
381 | |||
382 | build_command = [] | ||
383 | fs_type = prop_dict.get("fs_type", "") | ||
384 | |||
385 | fs_spans_partition = True | ||
386 | if fs_type.startswith("squash"): | ||
387 | fs_spans_partition = False | ||
388 | |||
389 | # Get a builder for creating an image that's to be verified by Verified Boot, | ||
390 | # or None if not applicable. | ||
391 | verity_image_builder = verity_utils.CreateVerityImageBuilder(prop_dict) | ||
392 | |||
393 | if (prop_dict.get("use_dynamic_partition_size") == "true" and | ||
394 | "partition_size" not in prop_dict): | ||
395 | # If partition_size is not defined, use output of `du' + reserved_size. | ||
396 | size = GetDiskUsage(in_dir) | ||
397 | logger.info( | ||
398 | "The tree size of %s is %d MB.", in_dir, size // BYTES_IN_MB) | ||
399 | # If not specified, give us 16MB margin for GetDiskUsage error ... | ||
400 | reserved_size = int(prop_dict.get("partition_reserved_size", BYTES_IN_MB * 16)) | ||
401 | partition_headroom = int(prop_dict.get("partition_headroom", 0)) | ||
402 | if fs_type.startswith("ext4") and partition_headroom > reserved_size: | ||
403 | reserved_size = partition_headroom | ||
404 | size += reserved_size | ||
405 | # Round this up to a multiple of 4K so that avbtool works | ||
406 | size = common.RoundUpTo4K(size) | ||
407 | if fs_type.startswith("ext"): | ||
408 | prop_dict["partition_size"] = str(size) | ||
409 | prop_dict["image_size"] = str(size) | ||
410 | if "extfs_inode_count" not in prop_dict: | ||
411 | prop_dict["extfs_inode_count"] = str(GetInodeUsage(in_dir)) | ||
412 | logger.info( | ||
413 | "First Pass based on estimates of %d MB and %s inodes.", | ||
414 | size // BYTES_IN_MB, prop_dict["extfs_inode_count"]) | ||
415 | BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config) | ||
416 | fs_dict = GetFilesystemCharacteristics(out_file) | ||
417 | os.remove(out_file) | ||
418 | block_size = int(fs_dict.get("Block size", "4096")) | ||
419 | free_size = int(fs_dict.get("Free blocks", "0")) * block_size | ||
420 | reserved_size = int(prop_dict.get("partition_reserved_size", 0)) | ||
421 | partition_headroom = int(fs_dict.get("partition_headroom", 0)) | ||
422 | if fs_type.startswith("ext4") and partition_headroom > reserved_size: | ||
423 | reserved_size = partition_headroom | ||
424 | if free_size <= reserved_size: | ||
425 | logger.info( | ||
426 | "Not worth reducing image %d <= %d.", free_size, reserved_size) | ||
427 | else: | ||
428 | size -= free_size | ||
429 | size += reserved_size | ||
430 | if block_size <= 4096: | ||
431 | size = common.RoundUpTo4K(size) | ||
432 | else: | ||
433 | size = ((size + block_size - 1) // block_size) * block_size | ||
434 | extfs_inode_count = prop_dict["extfs_inode_count"] | ||
435 | inodes = int(fs_dict.get("Inode count", extfs_inode_count)) | ||
436 | inodes -= int(fs_dict.get("Free inodes", "0")) | ||
437 | prop_dict["extfs_inode_count"] = str(inodes) | ||
438 | prop_dict["partition_size"] = str(size) | ||
439 | logger.info( | ||
440 | "Allocating %d Inodes for %s.", inodes, out_file) | ||
441 | if verity_image_builder: | ||
442 | size = verity_image_builder.CalculateDynamicPartitionSize(size) | ||
443 | prop_dict["partition_size"] = str(size) | ||
444 | logger.info( | ||
445 | "Allocating %d MB for %s.", size // BYTES_IN_MB, out_file) | ||
446 | |||
447 | prop_dict["image_size"] = prop_dict["partition_size"] | ||
448 | |||
449 | # Adjust the image size to make room for the hashes if this is to be verified. | ||
450 | if verity_image_builder: | ||
451 | max_image_size = verity_image_builder.CalculateMaxImageSize() | ||
452 | prop_dict["image_size"] = str(max_image_size) | ||
453 | |||
454 | mkfs_output = BuildImageMkfs(in_dir, prop_dict, out_file, target_out, fs_config) | ||
455 | |||
417 | # Check if there's enough headroom space available for ext4 image. | 456 | # Check if there's enough headroom space available for ext4 image. |
418 | if "partition_headroom" in prop_dict and fs_type.startswith("ext4"): | 457 | if "partition_headroom" in prop_dict and fs_type.startswith("ext4"): |
419 | CheckHeadroom(mkfs_output, prop_dict) | 458 | CheckHeadroom(mkfs_output, prop_dict) |
@@ -425,16 +464,6 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): | |||
425 | if verity_image_builder: | 464 | if verity_image_builder: |
426 | verity_image_builder.Build(out_file) | 465 | verity_image_builder.Build(out_file) |
427 | 466 | ||
428 | if run_e2fsck and prop_dict.get("skip_fsck") != "true": | ||
429 | unsparse_image = UnsparseImage(out_file, replace=False) | ||
430 | |||
431 | # Run e2fsck on the inflated image file | ||
432 | e2fsck_command = ["e2fsck", "-f", "-n", unsparse_image] | ||
433 | try: | ||
434 | common.RunAndCheckOutput(e2fsck_command) | ||
435 | finally: | ||
436 | os.remove(unsparse_image) | ||
437 | |||
438 | 467 | ||
439 | def ImagePropFromGlobalDict(glob_dict, mount_point): | 468 | def ImagePropFromGlobalDict(glob_dict, mount_point): |
440 | """Build an image property dictionary from the global dictionary. | 469 | """Build an image property dictionary from the global dictionary. |