summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorColin Cross2012-04-25 21:02:58 -0500
committerColin Cross2012-07-10 00:09:37 -0500
commit0c4c47f88dfc15cada154a1cf9b4db88b49890f0 (patch)
tree36b4a1635c149cbdb26347387185a5c824cd203a /libsparse
parent13a560659381b34ce3edbfa8dbe6c0aa6c076f20 (diff)
downloadplatform-system-core-0c4c47f88dfc15cada154a1cf9b4db88b49890f0.tar.gz
platform-system-core-0c4c47f88dfc15cada154a1cf9b4db88b49890f0.tar.xz
platform-system-core-0c4c47f88dfc15cada154a1cf9b4db88b49890f0.zip
libsparse: add sparse_file read and convert tools to use it
Abstract the logic from simg2img into libsparse, and add logic for reading a regular image into libsparse. simg2img then becomes a simple wrapper around libsparse. img2simg was not actually making the file sparse, it was using sparse files to create multiple files that could be pieced back together. Replace it with a simple wrapper around libsparse. Its functionality will be replaced by an simg2simg that can resparse a file into smaller chunks. Change-Id: I266f70e1c750454183ce46c71a7bb66bbb033a26
Diffstat (limited to 'libsparse')
-rw-r--r--libsparse/Android.mk7
-rw-r--r--libsparse/img2simg.c355
-rw-r--r--libsparse/include/sparse/sparse.h49
-rw-r--r--libsparse/simg2img.c258
-rw-r--r--libsparse/sparse_read.c509
5 files changed, 643 insertions, 535 deletions
diff --git a/libsparse/Android.mk b/libsparse/Android.mk
index d3eeae651..e83ee1cb0 100644
--- a/libsparse/Android.mk
+++ b/libsparse/Android.mk
@@ -7,7 +7,8 @@ libsparse_src_files := \
7 output_file.c \ 7 output_file.c \
8 sparse.c \ 8 sparse.c \
9 sparse_crc32.c \ 9 sparse_crc32.c \
10 sparse_err.c 10 sparse_err.c \
11 sparse_read.c
11 12
12include $(CLEAR_VARS) 13include $(CLEAR_VARS)
13 14
@@ -48,6 +49,7 @@ LOCAL_SRC_FILES := simg2img.c \
48 sparse_crc32.c 49 sparse_crc32.c
49LOCAL_MODULE := simg2img 50LOCAL_MODULE := simg2img
50LOCAL_MODULE_TAGS := debug 51LOCAL_MODULE_TAGS := debug
52LOCAL_STATIC_LIBRARIES := libsparse libz
51 53
52include $(BUILD_HOST_EXECUTABLE) 54include $(BUILD_HOST_EXECUTABLE)
53 55
@@ -57,6 +59,7 @@ LOCAL_SRC_FILES := simg2img.c \
57 sparse_crc32.c 59 sparse_crc32.c
58LOCAL_MODULE := simg2img 60LOCAL_MODULE := simg2img
59LOCAL_MODULE_TAGS := optional 61LOCAL_MODULE_TAGS := optional
62LOCAL_STATIC_LIBRARIES := libsparse libz
60 63
61include $(BUILD_EXECUTABLE) 64include $(BUILD_EXECUTABLE)
62 65
@@ -65,6 +68,7 @@ include $(CLEAR_VARS)
65LOCAL_SRC_FILES := img2simg.c 68LOCAL_SRC_FILES := img2simg.c
66LOCAL_MODULE := img2simg 69LOCAL_MODULE := img2simg
67LOCAL_MODULE_TAGS := debug 70LOCAL_MODULE_TAGS := debug
71LOCAL_STATIC_LIBRARIES := libsparse libz
68 72
69include $(BUILD_HOST_EXECUTABLE) 73include $(BUILD_HOST_EXECUTABLE)
70 74
@@ -73,6 +77,7 @@ include $(CLEAR_VARS)
73LOCAL_SRC_FILES := img2simg.c 77LOCAL_SRC_FILES := img2simg.c
74LOCAL_MODULE := img2simg 78LOCAL_MODULE := img2simg
75LOCAL_MODULE_TAGS := optional 79LOCAL_MODULE_TAGS := optional
80LOCAL_STATIC_LIBRARIES := libsparse libz
76 81
77include $(BUILD_EXECUTABLE) 82include $(BUILD_EXECUTABLE)
78 83
diff --git a/libsparse/img2simg.c b/libsparse/img2simg.c
index a1594df63..6b1caa506 100644
--- a/libsparse/img2simg.c
+++ b/libsparse/img2simg.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (C) 2010-2012 The Android Open Source Project 2 * Copyright (C) 2012 The Android Open Source Project
3 * 3 *
4 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License. 5 * you may not use this file except in compliance with the License.
@@ -14,314 +14,103 @@
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 16
17#define DEFAULT_BLOCK_SIZE "4K" 17#define _FILE_OFFSET_BITS 64
18#define DEFAULT_CHUNK_SIZE "64M" 18#define _LARGEFILE64_SOURCE 1
19#define DEFAULT_SUFFIX "%03d"
20 19
21#include "sparse_format.h"
22#if 0 /* endian.h is not on all platforms */
23# include <endian.h>
24#else
25 /* For now, just assume we're going to run on little-endian. */
26# define my_htole32(h) (h)
27# define my_htole16(h) (h)
28#endif
29#include <errno.h>
30#include <fcntl.h> 20#include <fcntl.h>
31#include <limits.h> 21#include <stdbool.h>
32#include <stdarg.h>
33#include <stddef.h>
34#include <stdio.h> 22#include <stdio.h>
35#include <stdlib.h> 23#include <stdlib.h>
36#include <string.h> 24#include <string.h>
37#include <unistd.h> 25#include <sys/types.h>
38#include <sys/stat.h> 26#include <sys/stat.h>
39#include <sys/types.h> 27#include <sys/types.h>
28#include <unistd.h>
40 29
41#define COPY_BUF_SIZE (1024*1024) 30#include <sparse/sparse.h>
42static char *copy_buf;
43
44static const char *progname(const char *argv0)
45{
46 const char *prog_name;
47 if ((prog_name = strrchr(argv0, '/')))
48 return(prog_name + 1); /* Advance beyond '/'. */
49 return(argv0); /* No '/' in argv0, use it as is. */
50}
51
52static void error_exit(const char *fmt, ...)
53{
54 va_list ap;
55 va_start(ap, fmt);
56 vfprintf(stderr, fmt, ap);
57 fputc('\n', stderr);
58 va_end(ap);
59
60 exit(EXIT_FAILURE);
61}
62
63static void usage(const char *argv0, const char *error_fmt, ...)
64{
65 fprintf(stderr,
66 "Usage: %s [OPTIONS] <raw_image_file>\n",
67 progname(argv0));
68 fprintf(stderr, "The <raw_image_file> will be split into as many sparse\n");
69 fprintf(stderr, "files as needed. Each sparse file will contain a single\n");
70 fprintf(stderr, "DONT CARE chunk to offset to the correct block and then\n");
71 fprintf(stderr, "a single RAW chunk containing a portion of the data from\n");
72 fprintf(stderr, "the raw image file. The sparse files will be named by\n");
73 fprintf(stderr, "appending a number to the name of the raw image file.\n");
74 fprintf(stderr, "\n");
75 fprintf(stderr, "OPTIONS (Defaults are enclosed by square brackets):\n");
76 fprintf(stderr, " -s SUFFIX Format appended number with SUFFIX [%s]\n",
77 DEFAULT_SUFFIX);
78 fprintf(stderr, " -B SIZE Use a block size of SIZE [%s]\n",
79 DEFAULT_BLOCK_SIZE);
80 fprintf(stderr, " -C SIZE Use a chunk size of SIZE [%s]\n",
81 DEFAULT_CHUNK_SIZE);
82 fprintf(stderr, "SIZE is a decimal integer that may optionally be\n");
83 fprintf(stderr, "followed by a suffix that specifies a multiplier for\n");
84 fprintf(stderr, "the integer:\n");
85 fprintf(stderr, " c 1 byte (the default when omitted)\n");
86 fprintf(stderr, " w 2 bytes\n");
87 fprintf(stderr, " b 512 bytes\n");
88 fprintf(stderr, " kB 1000 bytes\n");
89 fprintf(stderr, " K 1024 bytes\n");
90 fprintf(stderr, " MB 1000*1000 bytes\n");
91 fprintf(stderr, " M 1024*1024 bytes\n");
92 fprintf(stderr, " GB 1000*1000*1000 bytes\n");
93 fprintf(stderr, " G 1024*1024*1024 bytes\n");
94
95 if (error_fmt && *error_fmt)
96 {
97 fprintf(stderr, "\n");
98 va_list ap;
99 va_start(ap, error_fmt);
100 vfprintf(stderr, error_fmt, ap);
101 va_end(ap);
102 fprintf(stderr, "\n");
103 }
104
105 exit(EXIT_FAILURE);
106}
107
108static void cpy_file(int out_fd, char *out_path, int in_fd, char *in_path,
109 size_t len)
110{
111 ssize_t s, cpy_len = COPY_BUF_SIZE;
112
113 while (len) {
114 if (len < COPY_BUF_SIZE)
115 cpy_len = len;
116
117 s = read(in_fd, copy_buf, cpy_len);
118 if (s < 0)
119 error_exit("\"%s\": %s", in_path, strerror(errno));
120 if (!s)
121 error_exit("\"%s\": Unexpected EOF", in_path);
122
123 cpy_len = s;
124 31
125 s = write(out_fd, copy_buf, cpy_len); 32#ifndef O_BINARY
126 if (s < 0) 33#define O_BINARY 0
127 error_exit("\"%s\": %s", out_path, strerror(errno)); 34#endif
128 if (s != cpy_len)
129 error_exit("\"%s\": Short data write (%lu)", out_path,
130 (unsigned long)s);
131 35
132 len -= cpy_len; 36#if defined(__APPLE__) && defined(__MACH__)
133 } 37#define lseek64 lseek
134} 38#define off64_t off_t
39#endif
135 40
136static int parse_size(const char *size_str, size_t *size) 41void usage()
137{ 42{
138 static const size_t MAX_SIZE_T = ~(size_t)0; 43 fprintf(stderr, "Usage: img2simg <raw_image_file> <sparse_image_file> [<block_size>]\n");
139 size_t mult;
140 unsigned long long int value;
141 const char *end;
142 errno = 0;
143 value = strtoull(size_str, (char **)&end, 10);
144 if (errno != 0 || end == size_str || value > MAX_SIZE_T)
145 return -1;
146 if (*end == '\0') {
147 *size = value;
148 return 0;
149 }
150 if (!strcmp(end, "c"))
151 mult = 1;
152 else if (!strcmp(end, "w"))
153 mult = 2;
154 else if (!strcmp(end, "b"))
155 mult = 512;
156 else if (!strcmp(end, "kB"))
157 mult = 1000;
158 else if (!strcmp(end, "K"))
159 mult = 1024;
160 else if (!strcmp(end, "MB"))
161 mult = (size_t)1000*1000;
162 else if (!strcmp(end, "M"))
163 mult = (size_t)1024*1024;
164 else if (!strcmp(end, "GB"))
165 mult = (size_t)1000*1000*1000;
166 else if (!strcmp(end, "G"))
167 mult = (size_t)1024*1024*1024;
168 else
169 return -1;
170
171 if (value > MAX_SIZE_T / mult)
172 return -1;
173 *size = value * mult;
174 return 0;
175} 44}
176 45
177int main(int argc, char *argv[]) 46int main(int argc, char *argv[])
178{ 47{
179 char *suffix = DEFAULT_SUFFIX; 48 int in;
180 char *block_size_str = DEFAULT_BLOCK_SIZE; 49 int out;
181 char *chunk_size_str = DEFAULT_CHUNK_SIZE; 50 unsigned int i;
182 size_t block_size, chunk_size, blocks_per_chunk, to_write; 51 int ret;
183 char *in_path, *out_path, *out_fmt; 52 struct sparse_file *s;
184 int in_fd, out_fd; 53 unsigned int block_size = 4096;
185 struct stat in_st; 54 off64_t len;
186 off_t left_to_write; 55
187 struct { 56 if (argc < 3 || argc > 4) {
188 sparse_header_t sparse_hdr; 57 usage();
189 chunk_header_t dont_care_hdr; 58 exit(-1);
190 chunk_header_t raw_hdr;
191 } file_hdr;
192 unsigned int file_count;
193 ssize_t s;
194 int i;
195
196 /* Parse the command line. */
197 while ((i = getopt(argc, argv, "s:B:C:")) != -1)
198 {
199 switch (i) {
200 case 's':
201 suffix = optarg;
202 break;
203 case 'B':
204 block_size_str = optarg;
205 break;
206 case 'C':
207 chunk_size_str = optarg;
208 break;
209 default:
210 usage(argv[0], NULL);
211 break;
212 } 59 }
213 }
214
215 if (parse_size(block_size_str, &block_size))
216 usage(argv[0], "Can not parse \"%s\" as a block size.",
217 block_size_str);
218 if (block_size % 4096)
219 usage(argv[0], "Block size is not a multiple of 4096.");
220
221 if (parse_size(chunk_size_str, &chunk_size))
222 usage(argv[0], "Can not parse \"%s\" as a chunk size.",
223 chunk_size_str);
224 if (chunk_size % block_size)
225 usage(argv[0], "Chunk size is not a multiple of the block size.");
226 blocks_per_chunk = chunk_size / block_size;
227
228 if ((argc - optind) != 1)
229 usage(argv[0], "Missing or extra arguments.");
230 in_path = argv[optind];
231
232 /* Open the input file and validate it. */
233 if ((in_fd = open(in_path, O_RDONLY)) < 0)
234 error_exit("open \"%s\": %s", in_path, strerror(errno));
235 if (fstat(in_fd, &in_st))
236 error_exit("fstat \"%s\": %s", in_path, strerror(errno));
237 left_to_write = in_st.st_size;
238 if (left_to_write % block_size)
239 error_exit(
240 "\"%s\" size (%llu) is not a multiple of the block size (%llu).\n",
241 in_path,
242 (unsigned long long)left_to_write, (unsigned long long)block_size);
243 60
244 /* Get a buffer for copying the chunks. */ 61 if (argc == 4) {
245 if ((copy_buf = malloc(COPY_BUF_SIZE)) == 0) 62 block_size = atoi(argv[3]);
246 error_exit("malloc copy buffer: %s", strerror(errno)); 63 }
247
248 /* Get a buffer for a sprintf format to form output paths. */
249 if ((out_fmt = malloc(sizeof("%s") + strlen(suffix))) == 0)
250 error_exit("malloc format buffer: %s", strerror(errno));
251 out_fmt[0] = '%';
252 out_fmt[1] = 's';
253 strcpy(out_fmt + 2, suffix);
254
255 /* Get a buffer for an output path. */
256 i = snprintf(copy_buf, COPY_BUF_SIZE, out_fmt, in_path, UINT_MAX);
257 if (i >= COPY_BUF_SIZE)
258 error_exit("Ridulously long suffix: %s", suffix);
259 if ((out_path = malloc(i + 1)) == 0)
260 error_exit("malloc output path buffer: %s", strerror(errno));
261
262 /*
263 * Each file gets a sparse_header, a Don't Care chunk to offset to
264 * where the data belongs and then a Raw chunk with the actual data.
265 */
266 memset((void *)&file_hdr.sparse_hdr, 0, sizeof(file_hdr.sparse_hdr));
267 file_hdr.sparse_hdr.magic = my_htole32(SPARSE_HEADER_MAGIC);
268 file_hdr.sparse_hdr.major_version = my_htole16(1);
269 file_hdr.sparse_hdr.minor_version = my_htole16(0);
270 file_hdr.sparse_hdr.file_hdr_sz = my_htole16(sizeof(sparse_header_t));
271 file_hdr.sparse_hdr.chunk_hdr_sz = my_htole16(sizeof(chunk_header_t));
272 file_hdr.sparse_hdr.blk_sz = my_htole32(block_size);
273 /* The total_blks will be set in the file loop below. */
274 file_hdr.sparse_hdr.total_chunks = my_htole32(2);
275 file_hdr.sparse_hdr.image_checksum = my_htole32(0); /* Typically unused. */
276 64
277 memset((void *)&file_hdr.dont_care_hdr, 0, sizeof(file_hdr.dont_care_hdr)); 65 if (block_size < 1024 || block_size % 4 != 0) {
278 file_hdr.dont_care_hdr.chunk_type = my_htole16(CHUNK_TYPE_DONT_CARE); 66 usage();
279 /* The Don't Care's chunk_sz will be set in the file loop below. */ 67 exit(-1);
280 file_hdr.dont_care_hdr.total_sz = my_htole32(sizeof(chunk_header_t)); 68 }
281 69
282 memset((void *)&file_hdr.raw_hdr, 0, sizeof(file_hdr.raw_hdr)); 70 if (strcmp(argv[1], "-") == 0) {
283 file_hdr.raw_hdr.chunk_type = my_htole16(CHUNK_TYPE_RAW); 71 in = STDIN_FILENO;
284 file_hdr.raw_hdr.chunk_sz = my_htole32(blocks_per_chunk); 72 } else {
285 file_hdr.raw_hdr.total_sz = my_htole32(chunk_size + sizeof(chunk_header_t)); 73 in = open(argv[1], O_RDONLY | O_BINARY);
74 if (in < 0) {
75 fprintf(stderr, "Cannot open input file %s\n", argv[1]);
76 exit(-1);
77 }
78 }
286 79
287 /* Loop through writing chunk_size to each of the output files. */ 80 if (strcmp(argv[2], "-") == 0) {
288 to_write = chunk_size; 81 out = STDOUT_FILENO;
289 for (file_count = 1; left_to_write ; file_count++) { 82 } else {
290 /* Fix up the headers on the last block. */ 83 out = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664);
291 if (left_to_write < (off_t)chunk_size) { 84 if (out < 0) {
292 to_write = left_to_write; 85 fprintf(stderr, "Cannot open output file %s\n", argv[2]);
293 file_hdr.raw_hdr.chunk_sz = my_htole32(left_to_write / block_size); 86 exit(-1);
294 file_hdr.raw_hdr.total_sz = my_htole32(left_to_write 87 }
295 + sizeof(chunk_header_t));
296 } 88 }
297 89
298 /* Form the pathname for this output file and open it. */ 90 len = lseek64(in, 0, SEEK_END);
299 sprintf(out_path, out_fmt, in_path, file_count); 91 lseek64(in, 0, SEEK_SET);
300 if ((out_fd = creat(out_path, 0666)) < 0)
301 error_exit("\"%s\": %s", out_path, strerror(errno));
302 92
303 /* Update and write the headers to this output file. */ 93 s = sparse_file_new(block_size, len);
304 s = (file_count-1) * blocks_per_chunk; 94 if (!s) {
305 file_hdr.dont_care_hdr.chunk_sz = my_htole32(s); 95 fprintf(stderr, "Failed to create sparse file\n");
306 file_hdr.sparse_hdr.total_blks = my_htole32(s 96 exit(-1);
307 + (to_write / block_size)); 97 }
308 s = write(out_fd, (void *)&file_hdr, sizeof(file_hdr));
309 if (s < 0)
310 error_exit("\"%s\": %s", out_path, strerror(errno));
311 if (s != sizeof(file_hdr))
312 error_exit("\"%s\": Short write (%lu)", out_path, (unsigned long)s);
313 98
314 /* Copy this chunk from the input file to the output file. */ 99 sparse_file_verbose(s);
315 cpy_file(out_fd, out_path, in_fd, in_path, to_write); 100 ret = sparse_file_read(s, in, false, false);
101 if (ret) {
102 fprintf(stderr, "Failed to read file\n");
103 exit(-1);
104 }
316 105
317 /* Close this output file and update the amount left to write. */ 106 ret = sparse_file_write(s, out, false, true, false);
318 if (close(out_fd)) 107 if (ret) {
319 error_exit("close \"%s\": %s", out_path, strerror(errno)); 108 fprintf(stderr, "Failed to write sparse file\n");
320 left_to_write -= to_write; 109 exit(-1);
321 } 110 }
322 111
323 if (close(in_fd)) 112 close(in);
324 error_exit("close \"%s\": %s", in_path, strerror(errno)); 113 close(out);
325 114
326 exit(EXIT_SUCCESS); 115 exit(0);
327} 116}
diff --git a/libsparse/include/sparse/sparse.h b/libsparse/include/sparse/sparse.h
index 09a513725..ae5495599 100644
--- a/libsparse/include/sparse/sparse.h
+++ b/libsparse/include/sparse/sparse.h
@@ -158,6 +158,55 @@ int sparse_file_write(struct sparse_file *s, int fd, bool gz, bool sparse,
158 bool crc); 158 bool crc);
159 159
160/** 160/**
161 * sparse_file_read - read a file into a sparse file cookie
162 *
163 * @s - sparse file cookie
164 * @fd - file descriptor to read from
165 * @sparse - read a file in the Android sparse file format
166 * @crc - verify the crc of a file in the Android sparse file format
167 *
168 * Reads a file into a sparse file cookie. If sparse is true, the file is
169 * assumed to be in the Android sparse file format. If sparse is false, the
170 * file will be sparsed by looking for block aligned chunks of all zeros or
171 * another 32 bit value. If crc is true, the crc of the sparse file will be
172 * verified.
173 *
174 * Returns 0 on success, negative errno on error.
175 */
176int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc);
177
178/**
179 * sparse_file_import - import an existing sparse file
180 *
181 * @s - sparse file cookie
182 * @verbose - print verbose errors while reading the sparse file
183 * @crc - verify the crc of a file in the Android sparse file format
184 *
185 * Reads an existing sparse file into a sparse file cookie, recreating the same
186 * sparse cookie that was used to write it. If verbose is true, prints verbose
187 * errors when the sparse file is formatted incorrectly.
188 *
189 * Returns a new sparse file cookie on success, NULL on error.
190 */
191struct sparse_file *sparse_file_import(int fd, bool verbose, bool crc);
192
193/**
194 * sparse_file_import_auto - import an existing sparse or normal file
195 *
196 * @fd - file descriptor to read from
197 * @crc - verify the crc of a file in the Android sparse file format
198 *
199 * Reads an existing sparse or normal file into a sparse file cookie.
200 * Attempts to determine if the file is sparse or not by looking for the sparse
201 * file magic number in the first 4 bytes. If the file is not sparse, the file
202 * will be sparsed by looking for block aligned chunks of all zeros or another
203 * 32 bit value. If crc is true, the crc of the sparse file will be verified.
204 *
205 * Returns a new sparse file cookie on success, NULL on error.
206 */
207struct sparse_file *sparse_file_import_auto(int fd, bool crc);
208
209/**
161 * sparse_file_verbose - set a sparse file cookie to print verbose errors 210 * sparse_file_verbose - set a sparse file cookie to print verbose errors
162 * 211 *
163 * @s - sparse file cookie 212 * @s - sparse file cookie
diff --git a/libsparse/simg2img.c b/libsparse/simg2img.c
index 486b8054f..ab355838f 100644
--- a/libsparse/simg2img.c
+++ b/libsparse/simg2img.c
@@ -14,194 +14,36 @@
14 * limitations under the License. 14 * limitations under the License.
15 */ 15 */
16 16
17#define _FILE_OFFSET_BITS 64 17#include <sparse/sparse.h>
18#define _LARGEFILE64_SOURCE 1
19#include <sys/types.h>
20#include <unistd.h>
21
22#include "sparse_defs.h"
23#include "sparse_format.h"
24#include "sparse_crc32.h"
25 18
26#include <fcntl.h> 19#include <fcntl.h>
20#include <stdbool.h>
27#include <stdio.h> 21#include <stdio.h>
28#include <stdlib.h> 22#include <stdlib.h>
29#include <string.h> 23#include <string.h>
30#include <sys/types.h> 24#include <sys/types.h>
31#include <sys/stat.h> 25#include <sys/stat.h>
32#include <sys/types.h> 26#include <sys/types.h>
33#include <sys/mman.h>
34#include <unistd.h> 27#include <unistd.h>
35 28
36#define COPY_BUF_SIZE (1024*1024)
37u8 *copybuf;
38
39/* This will be malloc'ed with the size of blk_sz from the sparse file header */
40u8* zerobuf;
41
42#define SPARSE_HEADER_MAJOR_VER 1
43#define SPARSE_HEADER_LEN (sizeof(sparse_header_t))
44#define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
45
46void usage() 29void usage()
47{ 30{
48 fprintf(stderr, "Usage: simg2img <sparse_image_file> <raw_image_file>\n"); 31 fprintf(stderr, "Usage: simg2img <sparse_image_file> <raw_image_file>\n");
49} 32}
50 33
51static int read_all(int fd, void *buf, size_t len)
52{
53 size_t total = 0;
54 int ret;
55 char *ptr = buf;
56
57 while (total < len) {
58 ret = read(fd, ptr, len - total);
59
60 if (ret < 0)
61 return ret;
62
63 if (ret == 0)
64 return total;
65
66 ptr += ret;
67 total += ret;
68 }
69
70 return total;
71}
72
73static int write_all(int fd, void *buf, size_t len)
74{
75 size_t total = 0;
76 int ret;
77 char *ptr = buf;
78
79 while (total < len) {
80 ret = write(fd, ptr, len - total);
81
82 if (ret < 0)
83 return ret;
84
85 if (ret == 0)
86 return total;
87
88 ptr += ret;
89 total += ret;
90 }
91
92 return total;
93}
94
95int process_raw_chunk(int in, int out, u32 blocks, u32 blk_sz, u32 *crc32)
96{
97 u64 len = (u64)blocks * blk_sz;
98 int ret;
99 int chunk;
100
101 while (len) {
102 chunk = (len > COPY_BUF_SIZE) ? COPY_BUF_SIZE : len;
103 ret = read_all(in, copybuf, chunk);
104 if (ret != chunk) {
105 fprintf(stderr, "read returned an error copying a raw chunk: %d %d\n",
106 ret, chunk);
107 exit(-1);
108 }
109 *crc32 = sparse_crc32(*crc32, copybuf, chunk);
110 ret = write_all(out, copybuf, chunk);
111 if (ret != chunk) {
112 fprintf(stderr, "write returned an error copying a raw chunk\n");
113 exit(-1);
114 }
115 len -= chunk;
116 }
117
118 return blocks;
119}
120
121
122int process_fill_chunk(int in, int out, u32 blocks, u32 blk_sz, u32 *crc32)
123{
124 u64 len = (u64)blocks * blk_sz;
125 int ret;
126 int chunk;
127 u32 fill_val;
128 u32 *fillbuf;
129 unsigned int i;
130
131 /* Fill copy_buf with the fill value */
132 ret = read_all(in, &fill_val, sizeof(fill_val));
133 fillbuf = (u32 *)copybuf;
134 for (i = 0; i < (COPY_BUF_SIZE / sizeof(fill_val)); i++) {
135 fillbuf[i] = fill_val;
136 }
137
138 while (len) {
139 chunk = (len > COPY_BUF_SIZE) ? COPY_BUF_SIZE : len;
140 *crc32 = sparse_crc32(*crc32, copybuf, chunk);
141 ret = write_all(out, copybuf, chunk);
142 if (ret != chunk) {
143 fprintf(stderr, "write returned an error copying a raw chunk\n");
144 exit(-1);
145 }
146 len -= chunk;
147 }
148
149 return blocks;
150}
151
152int process_skip_chunk(int out, u32 blocks, u32 blk_sz, u32 *crc32)
153{
154 /* len needs to be 64 bits, as the sparse file specifies the skip amount
155 * as a 32 bit value of blocks.
156 */
157 u64 len = (u64)blocks * blk_sz;
158
159 lseek64(out, len, SEEK_CUR);
160
161 return blocks;
162}
163
164int process_crc32_chunk(int in, u32 crc32)
165{
166 u32 file_crc32;
167 int ret;
168
169 ret = read_all(in, &file_crc32, 4);
170 if (ret != 4) {
171 fprintf(stderr, "read returned an error copying a crc32 chunk\n");
172 exit(-1);
173 }
174
175 if (file_crc32 != crc32) {
176 fprintf(stderr, "computed crc32 of 0x%8.8x, expected 0x%8.8x\n",
177 crc32, file_crc32);
178 exit(-1);
179 }
180
181 return 0;
182}
183
184int main(int argc, char *argv[]) 34int main(int argc, char *argv[])
185{ 35{
186 int in; 36 int in;
187 int out; 37 int out;
188 unsigned int i; 38 unsigned int i;
189 sparse_header_t sparse_header;
190 chunk_header_t chunk_header;
191 u32 crc32 = 0;
192 u32 total_blocks = 0;
193 int ret; 39 int ret;
40 struct sparse_file *s;
194 41
195 if (argc != 3) { 42 if (argc != 3) {
196 usage(); 43 usage();
197 exit(-1); 44 exit(-1);
198 } 45 }
199 46
200 if ( (copybuf = malloc(COPY_BUF_SIZE)) == 0) {
201 fprintf(stderr, "Cannot malloc copy buf\n");
202 exit(-1);
203 }
204
205 if (strcmp(argv[1], "-") == 0) { 47 if (strcmp(argv[1], "-") == 0) {
206 in = STDIN_FILENO; 48 in = STDIN_FILENO;
207 } else { 49 } else {
@@ -220,102 +62,16 @@ int main(int argc, char *argv[])
220 } 62 }
221 } 63 }
222 64
223 ret = read_all(in, &sparse_header, sizeof(sparse_header)); 65 s = sparse_file_import(in, true, false);
224 if (ret != sizeof(sparse_header)) { 66 if (!s) {
225 fprintf(stderr, "Error reading sparse file header\n"); 67 fprintf(stderr, "Failed to read sparse file\n");
226 exit(-1);
227 }
228
229 if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
230 fprintf(stderr, "Bad magic\n");
231 exit(-1);
232 }
233
234 if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
235 fprintf(stderr, "Unknown major version number\n");
236 exit(-1);
237 }
238
239 if (sparse_header.file_hdr_sz > SPARSE_HEADER_LEN) {
240 /* Skip the remaining bytes in a header that is longer than
241 * we expected.
242 */
243 lseek64(in, sparse_header.file_hdr_sz - SPARSE_HEADER_LEN, SEEK_CUR);
244 }
245
246 if ( (zerobuf = malloc(sparse_header.blk_sz)) == 0) {
247 fprintf(stderr, "Cannot malloc zero buf\n");
248 exit(-1);
249 }
250
251 for (i=0; i<sparse_header.total_chunks; i++) {
252 ret = read_all(in, &chunk_header, sizeof(chunk_header));
253 if (ret != sizeof(chunk_header)) {
254 fprintf(stderr, "Error reading chunk header\n");
255 exit(-1);
256 }
257
258 if (sparse_header.chunk_hdr_sz > CHUNK_HEADER_LEN) {
259 /* Skip the remaining bytes in a header that is longer than
260 * we expected.
261 */
262 lseek64(in, sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN, SEEK_CUR);
263 }
264
265 switch (chunk_header.chunk_type) {
266 case CHUNK_TYPE_RAW:
267 if (chunk_header.total_sz != (sparse_header.chunk_hdr_sz +
268 (chunk_header.chunk_sz * sparse_header.blk_sz)) ) {
269 fprintf(stderr, "Bogus chunk size for chunk %d, type Raw\n", i);
270 exit(-1);
271 }
272 total_blocks += process_raw_chunk(in, out,
273 chunk_header.chunk_sz, sparse_header.blk_sz, &crc32);
274 break;
275 case CHUNK_TYPE_FILL:
276 if (chunk_header.total_sz != (sparse_header.chunk_hdr_sz + sizeof(u32)) ) {
277 fprintf(stderr, "Bogus chunk size for chunk %d, type Fill\n", i);
278 exit(-1);
279 }
280 total_blocks += process_fill_chunk(in, out,
281 chunk_header.chunk_sz, sparse_header.blk_sz, &crc32);
282 break;
283 case CHUNK_TYPE_DONT_CARE:
284 if (chunk_header.total_sz != sparse_header.chunk_hdr_sz) {
285 fprintf(stderr, "Bogus chunk size for chunk %d, type Dont Care\n", i);
286 exit(-1);
287 }
288 total_blocks += process_skip_chunk(out,
289 chunk_header.chunk_sz, sparse_header.blk_sz, &crc32);
290 break;
291 case CHUNK_TYPE_CRC32:
292 process_crc32_chunk(in, crc32);
293 break;
294 default:
295 fprintf(stderr, "Unknown chunk type 0x%4.4x\n", chunk_header.chunk_type);
296 }
297
298 }
299
300 /* If the last chunk was a skip, then the code just did a seek, but
301 * no write, and the file won't actually be the correct size. This
302 * will make the file the correct size. Make sure the offset is
303 * computed in 64 bits, and the function called can handle 64 bits.
304 */
305 if (ftruncate64(out, (u64)total_blocks * sparse_header.blk_sz)) {
306 fprintf(stderr, "Error calling ftruncate() to set the image size\n");
307 exit(-1); 68 exit(-1);
308 } 69 }
70 ret = sparse_file_write(s, out, false, false, false);
309 71
310 close(in); 72 close(in);
311 close(out); 73 close(out);
312 74
313 if (sparse_header.total_blks != total_blocks) {
314 fprintf(stderr, "Wrote %d blocks, expected to write %d blocks\n",
315 total_blocks, sparse_header.total_blks);
316 exit(-1);
317 }
318
319 exit(0); 75 exit(0);
320} 76}
321 77
diff --git a/libsparse/sparse_read.c b/libsparse/sparse_read.c
new file mode 100644
index 000000000..704bcfadd
--- /dev/null
+++ b/libsparse/sparse_read.c
@@ -0,0 +1,509 @@
1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define _GNU_SOURCE
18#define _FILE_OFFSET_BITS 64
19#define _LARGEFILE64_SOURCE 1
20
21#include <fcntl.h>
22#include <stdarg.h>
23#include <stdbool.h>
24#include <stdint.h>
25#include <stdio.h>
26#include <stdlib.h>
27#include <string.h>
28#include <unistd.h>
29
30#include <sparse/sparse.h>
31
32#include "sparse_crc32.h"
33#include "sparse_file.h"
34#include "sparse_format.h"
35
36#if defined(__APPLE__) && defined(__MACH__)
37#define lseek64 lseek
38#define off64_t off_t
39#endif
40
41#define SPARSE_HEADER_MAJOR_VER 1
42#define SPARSE_HEADER_LEN (sizeof(sparse_header_t))
43#define CHUNK_HEADER_LEN (sizeof(chunk_header_t))
44
45#define COPY_BUF_SIZE (1024U*1024U)
46static char *copybuf;
47
48#define min(a, b) \
49 ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
50
51static void verbose_error(bool verbose, int err, const char *fmt, ...)
52{
53 char *s = "";
54 char *at = "";
55 if (fmt) {
56 va_list argp;
57 int size;
58
59 va_start(argp, fmt);
60 size = vsnprintf(NULL, 0, fmt, argp);
61 va_end(argp);
62
63 if (size < 0) {
64 return;
65 }
66
67 at = malloc(size + 1);
68 if (at == NULL) {
69 return;
70 }
71
72 va_start(argp, fmt);
73 vsnprintf(at, size, fmt, argp);
74 va_end(argp);
75 at[size] = 0;
76 s = " at ";
77 }
78 if (verbose) {
79#ifndef USE_MINGW
80 if (err == -EOVERFLOW) {
81 sparse_print_verbose("EOF while reading file%s%s\n", s, at);
82 } else
83#endif
84 if (err == -EINVAL) {
85 sparse_print_verbose("Invalid sparse file format%s%s\n", s, at);
86 } else if (err == -ENOMEM) {
87 sparse_print_verbose("Failed allocation while reading file%s%s\n",
88 s, at);
89 } else {
90 sparse_print_verbose("Unknown error %d%s%s\n", err, s, at);
91 }
92 }
93 if (fmt) {
94 free(at);
95 }
96}
97
98static int process_raw_chunk(struct sparse_file *s, unsigned int chunk_size,
99 int fd, int64_t offset, unsigned int blocks, unsigned int block,
100 uint32_t *crc32)
101{
102 int ret;
103 int chunk;
104 unsigned int len = blocks * s->block_size;
105
106 if (chunk_size % s->block_size != 0) {
107 return -EINVAL;
108 }
109
110 if (chunk_size / s->block_size != blocks) {
111 return -EINVAL;
112 }
113
114 ret = sparse_file_add_fd(s, fd, offset, len, block);
115 if (ret < 0) {
116 return ret;
117 }
118
119 if (crc32) {
120 while (len) {
121 chunk = min(len, COPY_BUF_SIZE);
122 ret = read_all(fd, copybuf, chunk);
123 if (ret < 0) {
124 return ret;
125 }
126 *crc32 = sparse_crc32(*crc32, copybuf, chunk);
127 len -= chunk;
128 }
129 } else {
130 lseek64(fd, len, SEEK_CUR);
131 }
132
133 return 0;
134}
135
136static int process_fill_chunk(struct sparse_file *s, unsigned int chunk_size,
137 int fd, unsigned int blocks, unsigned int block, uint32_t *crc32)
138{
139 int ret;
140 int chunk;
141 int64_t len = (int64_t)blocks * s->block_size;
142 uint32_t fill_val;
143 uint32_t *fillbuf;
144 unsigned int i;
145
146 if (chunk_size != sizeof(fill_val)) {
147 return -EINVAL;
148 }
149
150 ret = read_all(fd, &fill_val, sizeof(fill_val));
151 if (ret < 0) {
152 return ret;
153 }
154
155 ret = sparse_file_add_fill(s, fill_val, len, block);
156 if (ret < 0) {
157 return ret;
158 }
159
160 if (crc32) {
161 /* Fill copy_buf with the fill value */
162 fillbuf = (uint32_t *)copybuf;
163 for (i = 0; i < (COPY_BUF_SIZE / sizeof(fill_val)); i++) {
164 fillbuf[i] = fill_val;
165 }
166
167 while (len) {
168 chunk = min(len, COPY_BUF_SIZE);
169 *crc32 = sparse_crc32(*crc32, copybuf, chunk);
170 len -= chunk;
171 }
172 }
173
174 return 0;
175}
176
177static int process_skip_chunk(struct sparse_file *s, unsigned int chunk_size,
178 int fd, unsigned int blocks, unsigned int block, uint32_t *crc32)
179{
180 int ret;
181 int chunk;
182 int64_t len = (int64_t)blocks * s->block_size;
183 uint32_t fill_val;
184 uint32_t *fillbuf;
185 unsigned int i;
186
187 if (chunk_size != 0) {
188 return -EINVAL;
189 }
190
191 if (crc32) {
192 memset(copybuf, 0, COPY_BUF_SIZE);
193
194 while (len) {
195 chunk = min(len, COPY_BUF_SIZE);
196 *crc32 = sparse_crc32(*crc32, copybuf, chunk);
197 len -= chunk;
198 }
199 }
200
201 return 0;
202}
203
204static int process_crc32_chunk(int fd, unsigned int chunk_size, uint32_t crc32)
205{
206 uint32_t file_crc32;
207 int ret;
208
209 if (chunk_size != sizeof(file_crc32)) {
210 return -EINVAL;
211 }
212
213 ret = read_all(fd, &file_crc32, sizeof(file_crc32));
214 if (ret < 0) {
215 return ret;
216 }
217
218 if (file_crc32 != crc32) {
219 return -EINVAL;
220 }
221
222 return 0;
223}
224
225static int process_chunk(struct sparse_file *s, int fd, off64_t offset,
226 unsigned int chunk_hdr_sz, chunk_header_t *chunk_header,
227 unsigned int cur_block, uint32_t *crc_ptr)
228{
229 int ret;
230 unsigned int chunk_data_size;
231
232 chunk_data_size = chunk_header->total_sz - chunk_hdr_sz;
233
234 switch (chunk_header->chunk_type) {
235 case CHUNK_TYPE_RAW:
236 ret = process_raw_chunk(s, chunk_data_size, fd, offset,
237 chunk_header->chunk_sz, cur_block, crc_ptr);
238 if (ret < 0) {
239 verbose_error(s->verbose, ret, "data block at %lld", offset);
240 return ret;
241 }
242 return chunk_header->chunk_sz;
243 case CHUNK_TYPE_FILL:
244 ret = process_fill_chunk(s, chunk_data_size, fd,
245 chunk_header->chunk_sz, cur_block, crc_ptr);
246 if (ret < 0) {
247 verbose_error(s->verbose, ret, "fill block at %lld", offset);
248 return ret;
249 }
250 return chunk_header->chunk_sz;
251 case CHUNK_TYPE_DONT_CARE:
252 ret = process_skip_chunk(s, chunk_data_size, fd,
253 chunk_header->chunk_sz, cur_block, crc_ptr);
254 if (chunk_data_size != 0) {
255 if (ret < 0) {
256 verbose_error(s->verbose, ret, "skip block at %lld", offset);
257 return ret;
258 }
259 }
260 return chunk_header->chunk_sz;
261 case CHUNK_TYPE_CRC32:
262 ret = process_crc32_chunk(fd, chunk_data_size, *crc_ptr);
263 if (ret < 0) {
264 verbose_error(s->verbose, -EINVAL, "crc block at %lld",
265 offset);
266 return ret;
267 }
268 return 0;
269 default:
270 verbose_error(s->verbose, -EINVAL, "unknown block %04X at %lld",
271 chunk_header->chunk_type, offset);
272 }
273
274 return 0;
275}
276
277static int sparse_file_read_sparse(struct sparse_file *s, int fd, bool crc)
278{
279 int ret;
280 unsigned int i;
281 sparse_header_t sparse_header;
282 chunk_header_t chunk_header;
283 uint32_t crc32 = 0;
284 uint32_t *crc_ptr = 0;
285 unsigned int cur_block = 0;
286 off64_t offset;
287
288 if (!copybuf) {
289 copybuf = malloc(COPY_BUF_SIZE);
290 }
291
292 if (!copybuf) {
293 return -ENOMEM;
294 }
295
296 if (crc) {
297 crc_ptr = &crc32;
298 }
299
300 ret = read_all(fd, &sparse_header, sizeof(sparse_header));
301 if (ret < 0) {
302 return ret;
303 }
304
305 if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
306 return -EINVAL;
307 }
308
309 if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
310 return -EINVAL;
311 }
312
313 if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
314 return -EINVAL;
315 }
316
317 if (sparse_header.chunk_hdr_sz < sizeof(chunk_header)) {
318 return -EINVAL;
319 }
320
321 if (sparse_header.file_hdr_sz > SPARSE_HEADER_LEN) {
322 /* Skip the remaining bytes in a header that is longer than
323 * we expected.
324 */
325 lseek64(fd, sparse_header.file_hdr_sz - SPARSE_HEADER_LEN, SEEK_CUR);
326 }
327
328 for (i = 0; i < sparse_header.total_chunks; i++) {
329 ret = read_all(fd, &chunk_header, sizeof(chunk_header));
330 if (ret < 0) {
331 return ret;
332 }
333
334 if (sparse_header.chunk_hdr_sz > CHUNK_HEADER_LEN) {
335 /* Skip the remaining bytes in a header that is longer than
336 * we expected.
337 */
338 lseek64(fd, sparse_header.chunk_hdr_sz - CHUNK_HEADER_LEN, SEEK_CUR);
339 }
340
341 offset = lseek64(fd, 0, SEEK_CUR);
342
343 ret = process_chunk(s, fd, offset, sparse_header.chunk_hdr_sz, &chunk_header,
344 cur_block, crc_ptr);
345 if (ret < 0) {
346 return ret;
347 }
348
349 cur_block += ret;
350 }
351
352 if (sparse_header.total_blks != cur_block) {
353 return -EINVAL;
354 }
355
356 return 0;
357}
358
359static int sparse_file_read_normal(struct sparse_file *s, int fd)
360{
361 int ret;
362 uint32_t *buf = malloc(s->block_size);
363 unsigned int block = 0;
364 int64_t remain = s->len;
365 int64_t offset = 0;
366 unsigned int to_read;
367 char *ptr;
368 unsigned int i;
369 bool sparse_block;
370
371 if (!buf) {
372 return -ENOMEM;
373 }
374
375 while (remain > 0) {
376 to_read = min(remain, s->block_size);
377 ret = read_all(fd, buf, to_read);
378 if (ret < 0) {
379 error("failed to read sparse file");
380 return ret;
381 }
382
383 if (to_read == s->block_size) {
384 sparse_block = true;
385 for (i = 1; i < s->block_size / sizeof(uint32_t); i++) {
386 if (buf[0] != buf[i]) {
387 sparse_block = false;
388 break;
389 }
390 }
391 } else {
392 sparse_block = false;
393 }
394
395 if (sparse_block) {
396 /* TODO: add flag to use skip instead of fill for buf[0] == 0 */
397 sparse_file_add_fill(s, buf[0], to_read, block);
398 } else {
399 sparse_file_add_fd(s, fd, offset, to_read, block);
400 }
401
402 remain -= to_read;
403 offset += to_read;
404 block++;
405 }
406
407 return 0;
408}
409
410int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc)
411{
412 if (crc && !sparse) {
413 return -EINVAL;
414 }
415
416 if (sparse) {
417 return sparse_file_read_sparse(s, fd, crc);
418 } else {
419 return sparse_file_read_normal(s, fd);
420 }
421}
422
423struct sparse_file *sparse_file_import(int fd, bool verbose, bool crc)
424{
425 int ret;
426 sparse_header_t sparse_header;
427 int64_t len;
428 struct sparse_file *s;
429
430 ret = read_all(fd, &sparse_header, sizeof(sparse_header));
431 if (ret < 0) {
432 verbose_error(verbose, ret, "header");
433 return NULL;
434 }
435
436 if (sparse_header.magic != SPARSE_HEADER_MAGIC) {
437 verbose_error(verbose, -EINVAL, "header magic");
438 return NULL;
439 }
440
441 if (sparse_header.major_version != SPARSE_HEADER_MAJOR_VER) {
442 verbose_error(verbose, -EINVAL, "header major version");
443 return NULL;
444 }
445
446 if (sparse_header.file_hdr_sz < SPARSE_HEADER_LEN) {
447 return NULL;
448 }
449
450 if (sparse_header.chunk_hdr_sz < sizeof(chunk_header_t)) {
451 return NULL;
452 }
453
454 len = (int64_t)sparse_header.total_blks * sparse_header.blk_sz;
455 s = sparse_file_new(sparse_header.blk_sz, len);
456 if (!s) {
457 verbose_error(verbose, -EINVAL, NULL);
458 return NULL;
459 }
460
461 ret = lseek64(fd, 0, SEEK_SET);
462 if (ret < 0) {
463 verbose_error(verbose, ret, "seeking");
464 sparse_file_destroy(s);
465 return NULL;
466 }
467
468 s->verbose = verbose;
469
470 ret = sparse_file_read(s, fd, true, crc);
471 if (ret < 0) {
472 sparse_file_destroy(s);
473 return NULL;
474 }
475
476 return s;
477}
478
479struct sparse_file *sparse_file_import_auto(int fd, bool crc)
480{
481 struct sparse_file *s;
482 int64_t len;
483 int ret;
484
485 s = sparse_file_import(fd, true, crc);
486 if (s) {
487 return s;
488 }
489
490 len = lseek64(fd, 0, SEEK_END);
491 if (len < 0) {
492 return NULL;
493 }
494
495 lseek64(fd, 0, SEEK_SET);
496
497 s = sparse_file_new(4096, len);
498 if (!s) {
499 return NULL;
500 }
501
502 ret = sparse_file_read_normal(s, fd);
503 if (ret < 0) {
504 sparse_file_destroy(s);
505 return NULL;
506 }
507
508 return s;
509}