summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorColin Cross2012-05-02 17:18:22 -0500
committerColin Cross2012-07-10 00:09:37 -0500
commitbdc6d39ed6c09199a5d806f29b71b44cbb27c5c2 (patch)
tree63fb3d31d9afc8ccc9c5a6ec96ddfcadb6a96f69 /libsparse
parent1e17b313a6257b7b5081e178e81435c09d60378e (diff)
downloadplatform-system-core-bdc6d39ed6c09199a5d806f29b71b44cbb27c5c2.tar.gz
platform-system-core-bdc6d39ed6c09199a5d806f29b71b44cbb27c5c2.tar.xz
platform-system-core-bdc6d39ed6c09199a5d806f29b71b44cbb27c5c2.zip
libsparse: add function to resparse a file and a utility to use it
Add sparse_file_repsarse, which splits chunks in an existing sparse file such that the maximum size of a chunk, plus a header and footer, is smaller than the given size. This will allow multiple smaller sparse files to result in the same data as a large sparse file. Change-Id: I177abdb958a23d5afd394ff265c5b0c6a3ff22fa
Diffstat (limited to 'libsparse')
-rw-r--r--libsparse/Android.mk9
-rw-r--r--libsparse/backed_block.c87
-rw-r--r--libsparse/backed_block.h6
-rw-r--r--libsparse/include/sparse/sparse.h14
-rw-r--r--libsparse/simg2img.c56
-rw-r--r--libsparse/simg2simg.c115
-rw-r--r--libsparse/sparse.c99
-rw-r--r--libsparse/sparse_defs.h1
8 files changed, 365 insertions, 22 deletions
diff --git a/libsparse/Android.mk b/libsparse/Android.mk
index e83ee1cb0..69b52c387 100644
--- a/libsparse/Android.mk
+++ b/libsparse/Android.mk
@@ -83,6 +83,15 @@ include $(BUILD_EXECUTABLE)
83 83
84include $(CLEAR_VARS) 84include $(CLEAR_VARS)
85 85
86LOCAL_SRC_FILES := simg2simg.c
87LOCAL_MODULE := simg2simg
88LOCAL_MODULE_TAGS := debug
89LOCAL_STATIC_LIBRARIES := libsparse libz
90
91include $(BUILD_HOST_EXECUTABLE)
92
93include $(CLEAR_VARS)
94
86LOCAL_MODULE := simg_dump.py 95LOCAL_MODULE := simg_dump.py
87LOCAL_MODULE_TAGS := debug 96LOCAL_MODULE_TAGS := debug
88LOCAL_SRC_FILES := simg_dump.py 97LOCAL_SRC_FILES := simg_dump.py
diff --git a/libsparse/backed_block.c b/libsparse/backed_block.c
index 629fc284f..dfb217b30 100644
--- a/libsparse/backed_block.c
+++ b/libsparse/backed_block.c
@@ -141,6 +141,52 @@ void backed_block_list_destroy(struct backed_block_list *bbl)
141 free(bbl); 141 free(bbl);
142} 142}
143 143
144void backed_block_list_move(struct backed_block_list *from,
145 struct backed_block_list *to, struct backed_block *start,
146 struct backed_block *end)
147{
148 struct backed_block *bb;
149
150 if (start == NULL) {
151 start = from->data_blocks;
152 }
153
154 if (!end) {
155 for (end = start; end && end->next; end = end->next)
156 ;
157 }
158
159 if (start == NULL || end == NULL) {
160 return;
161 }
162
163 from->last_used = NULL;
164 to->last_used = NULL;
165 if (from->data_blocks == start) {
166 from->data_blocks = end->next;
167 } else {
168 for (bb = from->data_blocks; bb; bb = bb->next) {
169 if (bb->next == start) {
170 bb->next = end->next;
171 break;
172 }
173 }
174 }
175
176 if (!to->data_blocks) {
177 to->data_blocks = start;
178 end->next = NULL;
179 } else {
180 for (bb = to->data_blocks; bb; bb = bb->next) {
181 if (!bb->next || bb->next->block > start->block) {
182 end->next = bb->next;
183 bb->next = start;
184 break;
185 }
186 }
187 }
188}
189
144/* may free b */ 190/* may free b */
145static int merge_bb(struct backed_block_list *bbl, 191static int merge_bb(struct backed_block_list *bbl,
146 struct backed_block *a, struct backed_block *b) 192 struct backed_block *a, struct backed_block *b)
@@ -311,3 +357,44 @@ int backed_block_add_fd(struct backed_block_list *bbl, int fd, int64_t offset,
311 357
312 return queue_bb(bbl, bb); 358 return queue_bb(bbl, bb);
313} 359}
360
361int backed_block_split(struct backed_block_list *bbl, struct backed_block *bb,
362 unsigned int max_len)
363{
364 struct backed_block *new_bb;
365
366 max_len = ALIGN_DOWN(max_len, bbl->block_size);
367
368 if (bb->len <= max_len) {
369 return 0;
370 }
371
372 new_bb = malloc(sizeof(struct backed_block));
373 if (bb == NULL) {
374 return -ENOMEM;
375 }
376
377 *new_bb = *bb;
378
379 new_bb->len = bb->len - max_len;
380 new_bb->block = bb->block + max_len / bbl->block_size;
381 new_bb->next = bb->next;
382 bb->next = new_bb;
383 bb->len = max_len;
384
385 switch (bb->type) {
386 case BACKED_BLOCK_DATA:
387 new_bb->data.data = (char *)bb->data.data + max_len;
388 break;
389 case BACKED_BLOCK_FILE:
390 new_bb->file.offset += max_len;
391 break;
392 case BACKED_BLOCK_FD:
393 new_bb->fd.offset += max_len;
394 break;
395 case BACKED_BLOCK_FILL:
396 break;
397 }
398
399 return 0;
400}
diff --git a/libsparse/backed_block.h b/libsparse/backed_block.h
index 692691712..1a159be1e 100644
--- a/libsparse/backed_block.h
+++ b/libsparse/backed_block.h
@@ -48,6 +48,8 @@ int backed_block_fd(struct backed_block *bb);
48int64_t backed_block_file_offset(struct backed_block *bb); 48int64_t backed_block_file_offset(struct backed_block *bb);
49uint32_t backed_block_fill_val(struct backed_block *bb); 49uint32_t backed_block_fill_val(struct backed_block *bb);
50enum backed_block_type backed_block_type(struct backed_block *bb); 50enum backed_block_type backed_block_type(struct backed_block *bb);
51int backed_block_split(struct backed_block_list *bbl, struct backed_block *bb,
52 unsigned int max_len);
51 53
52struct backed_block *backed_block_iter_new(struct backed_block_list *bbl); 54struct backed_block *backed_block_iter_new(struct backed_block_list *bbl);
53struct backed_block *backed_block_iter_next(struct backed_block *bb); 55struct backed_block *backed_block_iter_next(struct backed_block *bb);
@@ -55,4 +57,8 @@ struct backed_block *backed_block_iter_next(struct backed_block *bb);
55struct backed_block_list *backed_block_list_new(unsigned int block_size); 57struct backed_block_list *backed_block_list_new(unsigned int block_size);
56void backed_block_list_destroy(struct backed_block_list *bbl); 58void backed_block_list_destroy(struct backed_block_list *bbl);
57 59
60void backed_block_list_move(struct backed_block_list *from,
61 struct backed_block_list *to, struct backed_block *start,
62 struct backed_block *end);
63
58#endif 64#endif
diff --git a/libsparse/include/sparse/sparse.h b/libsparse/include/sparse/sparse.h
index b2152270b..fe003f61b 100644
--- a/libsparse/include/sparse/sparse.h
+++ b/libsparse/include/sparse/sparse.h
@@ -227,6 +227,20 @@ struct sparse_file *sparse_file_import(int fd, bool verbose, bool crc);
227 */ 227 */
228struct sparse_file *sparse_file_import_auto(int fd, bool crc); 228struct sparse_file *sparse_file_import_auto(int fd, bool crc);
229 229
230/** sparse_file_resparse - rechunk an existing sparse file into smaller files
231 *
232 * @in_s - sparse file cookie of the existing sparse file
233 * @max_len - maximum file size
234 * @out_s - array of sparse file cookies
235 * @out_s_count - size of out_s array
236 *
237 * Splits chunks of an existing sparse file into smaller sparse files such that
238 * each sparse file is less than max_len. Returns the number of sparse_files
239 * that would have been written to out_s if out_s were big enough.
240 */
241int sparse_file_resparse(struct sparse_file *in_s, unsigned int max_len,
242 struct sparse_file **out_s, int out_s_count);
243
230/** 244/**
231 * sparse_file_verbose - set a sparse file cookie to print verbose errors 245 * sparse_file_verbose - set a sparse file cookie to print verbose errors
232 * 246 *
diff --git a/libsparse/simg2img.c b/libsparse/simg2img.c
index ab355838f..95e9b5bea 100644
--- a/libsparse/simg2img.c
+++ b/libsparse/simg2img.c
@@ -26,50 +26,62 @@
26#include <sys/types.h> 26#include <sys/types.h>
27#include <unistd.h> 27#include <unistd.h>
28 28
29#ifndef O_BINARY
30#define O_BINARY 0
31#endif
32
29void usage() 33void usage()
30{ 34{
31 fprintf(stderr, "Usage: simg2img <sparse_image_file> <raw_image_file>\n"); 35 fprintf(stderr, "Usage: simg2img <sparse_image_files> <raw_image_file>\n");
32} 36}
33 37
34int main(int argc, char *argv[]) 38int main(int argc, char *argv[])
35{ 39{
36 int in; 40 int in;
37 int out; 41 int out;
38 unsigned int i; 42 int i;
39 int ret; 43 int ret;
40 struct sparse_file *s; 44 struct sparse_file *s;
41 45
42 if (argc != 3) { 46 if (argc < 3) {
43 usage(); 47 usage();
44 exit(-1); 48 exit(-1);
45 } 49 }
46 50
47 if (strcmp(argv[1], "-") == 0) { 51 out = open(argv[argc - 1], O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664);
48 in = STDIN_FILENO; 52 if (out < 0) {
49 } else { 53 fprintf(stderr, "Cannot open output file %s\n", argv[argc - 1]);
50 if ((in = open(argv[1], O_RDONLY)) == 0) { 54 exit(-1);
51 fprintf(stderr, "Cannot open input file %s\n", argv[1]);
52 exit(-1);
53 }
54 } 55 }
55 56
56 if (strcmp(argv[2], "-") == 0) { 57 for (i = 1; i < argc - 1; i++) {
57 out = STDOUT_FILENO; 58 if (strcmp(argv[i], "-") == 0) {
58 } else { 59 in = STDIN_FILENO;
59 if ((out = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC, 0666)) == 0) { 60 } else {
60 fprintf(stderr, "Cannot open output file %s\n", argv[2]); 61 in = open(argv[i], O_RDONLY | O_BINARY);
62 if (in < 0) {
63 fprintf(stderr, "Cannot open input file %s\n", argv[i]);
64 exit(-1);
65 }
66 }
67
68 s = sparse_file_import(in, true, false);
69 if (!s) {
70 fprintf(stderr, "Failed to read sparse file\n");
61 exit(-1); 71 exit(-1);
62 } 72 }
63 }
64 73
65 s = sparse_file_import(in, true, false); 74 lseek(out, SEEK_SET, 0);
66 if (!s) { 75
67 fprintf(stderr, "Failed to read sparse file\n"); 76 ret = sparse_file_write(s, out, false, false, false);
68 exit(-1); 77 if (ret < 0) {
78 fprintf(stderr, "Cannot write output file\n");
79 exit(-1);
80 }
81 sparse_file_destroy(s);
82 close(in);
69 } 83 }
70 ret = sparse_file_write(s, out, false, false, false);
71 84
72 close(in);
73 close(out); 85 close(out);
74 86
75 exit(0); 87 exit(0);
diff --git a/libsparse/simg2simg.c b/libsparse/simg2simg.c
new file mode 100644
index 000000000..5f9ccf678
--- /dev/null
+++ b/libsparse/simg2simg.c
@@ -0,0 +1,115 @@
1/*
2 * Copyright (C) 2012 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#define _FILE_OFFSET_BITS 64
18#define _LARGEFILE64_SOURCE 1
19#define _GNU_SOURCE
20
21#include <fcntl.h>
22#include <stdbool.h>
23#include <stdio.h>
24#include <stdlib.h>
25#include <string.h>
26#include <sys/types.h>
27#include <sys/stat.h>
28#include <sys/types.h>
29#include <unistd.h>
30
31#include <sparse/sparse.h>
32
33#ifndef O_BINARY
34#define O_BINARY 0
35#endif
36
37void usage()
38{
39 fprintf(stderr, "Usage: simg2simg <sparse image file> <sparse_image_file> <max_size>\n");
40}
41
42int main(int argc, char *argv[])
43{
44 int in;
45 int out;
46 int i;
47 int ret;
48 struct sparse_file *s;
49 int64_t max_size;
50 struct sparse_file **out_s;
51 int files;
52 char filename[4096];
53
54 if (argc != 4) {
55 usage();
56 exit(-1);
57 }
58
59 max_size = atoll(argv[3]);
60
61 in = open(argv[1], O_RDONLY | O_BINARY);
62 if (in < 0) {
63 fprintf(stderr, "Cannot open input file %s\n", argv[1]);
64 exit(-1);
65 }
66
67 s = sparse_file_import(in, true, false);
68 if (!s) {
69 fprintf(stderr, "Failed to import sparse file\n");
70 exit(-1);
71 }
72
73 files = sparse_file_resparse(s, max_size, NULL, 0);
74 if (files < 0) {
75 fprintf(stderr, "Failed to resparse\n");
76 exit(-1);
77 }
78
79 out_s = calloc(sizeof(struct sparse_file *), files);
80 if (!out_s) {
81 fprintf(stderr, "Failed to allocate sparse file array\n");
82 exit(-1);
83 }
84
85 files = sparse_file_resparse(s, max_size, out_s, files);
86 if (files < 0) {
87 fprintf(stderr, "Failed to resparse\n");
88 exit(-1);
89 }
90
91 for (i = 0; i < files; i++) {
92 ret = snprintf(filename, sizeof(filename), "%s.%d", argv[2], i);
93 if (ret >= (int)sizeof(filename)) {
94 fprintf(stderr, "Filename too long\n");
95 exit(-1);
96 }
97
98 out = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664);
99 if (out < 0) {
100 fprintf(stderr, "Cannot open output file %s\n", argv[2]);
101 exit(-1);
102 }
103
104 ret = sparse_file_write(out_s[i], out, false, true, false);
105 if (ret) {
106 fprintf(stderr, "Failed to write sparse file\n");
107 exit(-1);
108 }
109 close(out);
110 }
111
112 close(in);
113
114 exit(0);
115}
diff --git a/libsparse/sparse.c b/libsparse/sparse.c
index c560ec93c..77f02fc49 100644
--- a/libsparse/sparse.c
+++ b/libsparse/sparse.c
@@ -24,6 +24,7 @@
24#include "output_file.h" 24#include "output_file.h"
25#include "backed_block.h" 25#include "backed_block.h"
26#include "sparse_defs.h" 26#include "sparse_defs.h"
27#include "sparse_format.h"
27 28
28struct sparse_file *sparse_file_new(unsigned int block_size, int64_t len) 29struct sparse_file *sparse_file_new(unsigned int block_size, int64_t len)
29{ 30{
@@ -188,6 +189,104 @@ int sparse_file_callback(struct sparse_file *s, bool sparse, bool crc,
188 return ret; 189 return ret;
189} 190}
190 191
192static int out_counter_write(void *priv, const void *data, int len)
193{
194 int64_t *count = priv;
195 *count += len;
196 return 0;
197}
198
199static struct backed_block *move_chunks_up_to_len(struct sparse_file *from,
200 struct sparse_file *to, unsigned int len)
201{
202 int64_t count = 0;
203 struct output_file *out_counter;
204 struct backed_block *last_bb = NULL;
205 struct backed_block *bb;
206 struct backed_block *start;
207 int64_t file_len = 0;
208
209 /*
210 * overhead is sparse file header, initial skip chunk, split chunk, end
211 * skip chunk, and crc chunk.
212 */
213 int overhead = sizeof(sparse_header_t) + 4 * sizeof(chunk_header_t) +
214 sizeof(uint32_t);
215 len -= overhead;
216
217 start = backed_block_iter_new(from->backed_block_list);
218 out_counter = open_output_callback(out_counter_write, &count,
219 to->block_size, to->len, false, true, 0, false);
220 if (!out_counter) {
221 return NULL;
222 }
223
224 for (bb = start; bb; bb = backed_block_iter_next(bb)) {
225 count = 0;
226 /* will call out_counter_write to update count */
227 sparse_file_write_block(out_counter, bb);
228 if (file_len + count > len) {
229 /*
230 * If the remaining available size is more than 1/8th of the
231 * requested size, split the chunk. Results in sparse files that
232 * are at least 7/8ths of the requested size
233 */
234 if (!last_bb || (len - file_len > (len / 8))) {
235 backed_block_split(from->backed_block_list, bb, len - file_len);
236 last_bb = bb;
237 }
238 goto out;
239 }
240 file_len += count;
241 last_bb = bb;
242 }
243
244out:
245 backed_block_list_move(from->backed_block_list,
246 to->backed_block_list, start, last_bb);
247
248 close_output_file(out_counter);
249
250 return bb;
251}
252
253int sparse_file_resparse(struct sparse_file *in_s, unsigned int max_len,
254 struct sparse_file **out_s, int out_s_count)
255{
256 struct backed_block *bb;
257 unsigned int overhead;
258 struct sparse_file *s;
259 struct sparse_file *tmp;
260 int c = 0;
261
262 tmp = sparse_file_new(in_s->block_size, in_s->len);
263 if (!tmp) {
264 return -ENOMEM;
265 }
266
267 do {
268 s = sparse_file_new(in_s->block_size, in_s->len);
269
270 bb = move_chunks_up_to_len(in_s, s, max_len);
271
272 if (c < out_s_count) {
273 out_s[c] = s;
274 } else {
275 backed_block_list_move(s->backed_block_list, tmp->backed_block_list,
276 NULL, NULL);
277 sparse_file_destroy(s);
278 }
279 c++;
280 } while (bb);
281
282 backed_block_list_move(tmp->backed_block_list, in_s->backed_block_list,
283 NULL, NULL);
284
285 sparse_file_destroy(tmp);
286
287 return c;
288}
289
191void sparse_file_verbose(struct sparse_file *s) 290void sparse_file_verbose(struct sparse_file *s)
192{ 291{
193 s->verbose = true; 292 s->verbose = true;
diff --git a/libsparse/sparse_defs.h b/libsparse/sparse_defs.h
index 9f32d592e..b99cfd584 100644
--- a/libsparse/sparse_defs.h
+++ b/libsparse/sparse_defs.h
@@ -41,6 +41,7 @@ typedef unsigned char u8;
41 41
42#define DIV_ROUND_UP(x, y) (((x) + (y) - 1)/(y)) 42#define DIV_ROUND_UP(x, y) (((x) + (y) - 1)/(y))
43#define ALIGN(x, y) ((y) * DIV_ROUND_UP((x), (y))) 43#define ALIGN(x, y) ((y) * DIV_ROUND_UP((x), (y)))
44#define ALIGN_DOWN(x, y) ((y) * ((x) / (y)))
44 45
45#define error(fmt, args...) do { fprintf(stderr, "error: %s: " fmt "\n", __func__, ## args); } while (0) 46#define error(fmt, args...) do { fprintf(stderr, "error: %s: " fmt "\n", __func__, ## args); } while (0)
46#define error_errno(s, args...) error(s ": %s", ##args, strerror(errno)) 47#define error_errno(s, args...) error(s ": %s", ##args, strerror(errno))