1. Sparse image upgrade background
The commonly used system image format is the original image, that is, the RAW format. The image size is relatively large, and it is time-consuming to burn or upgrade the firmware, and it consumes more traffic during the upgrade process of mobile devices. For this reason, the original image is described sparsely, which can greatly reduce the size of the image, saving time and traffic.
2. The principle of sparse mirroring
1. The concept of sparse mirroring
Original image: that is, raw image, a complete ext4 partition image, including many invalid padding areas with all zeros
Sparse image: that is, sparse image, which describes the raw ext4 sparsely, so the size is relatively small, and the number of files in the production directory is counted, without all zero padding
2. Sparse image format
Sparse image data format: first, the sparse_header occupies 28 bytes, and then the chunk_header of 12 bytes. The type of the chunk_header also determines the following data. If the read data is 0xCAC1, it means that it is followed by its own raw_data. If it is 0xCAC3, then num It is 0, and then 0xCAC2 means that the content of 4byte is filled in the back.
3. Realize the sparse image upgrade solution
Version baseline:
OpenAtom OpenHarmony (hereinafter referred to as "OpenHarmony") 3.1 Release
code path:
https://gitee.com/openharmony/docs/blob/master/zh-cn/release-notes/OpenHarmony-v3.1-release.md
1. Sparse image burning
(1) Generate a sparse format image
There are 2 ways to generate a sparse image:
1) Modify the file build/ohos_var.gni, sparse_image=true
2) Add the --sparse-image field to the compilation command, such as ./build.sh --product-name=xxx --sparse-image
(2) Add sparse format conversion tool
Add the file img2simg in the directory build/ohos/images/mkimage. This tool is used to convert the raw image to sparse format after compilation, and set the permission to 777.
(3) Compiled image comparison
编译出的镜像格式为sparse格式，镜像大小相比raw格式明显变小。
（4）烧录稀疏镜像
烧录稀疏镜像方法和烧录原始镜像方法一致。
稀疏镜像本身是不能直接挂载的，在烧录过程中通过uboot将稀疏格式镜像还原为原始镜像，然后写到磁盘中，系统启动后可挂载对应的镜像。
2、稀疏镜像升级
升级包采用稀疏镜像制作。
（1）修改升级包制作工具
官方升级包工具不支持生成稀疏镜像的升级包，修改升级包工具，生成稀疏格式的升级包。.\base\update\packaging_tools\image_class.py
按照上图所示注释代码
（2）生成稀疏镜像升级包
和全量镜像升级包制作方法一致。
参考：
（3）适配updater组件中稀疏镜像功能
● 增加写稀疏镜像分支
.\base\update\updater\services\applypatch\data_writer.cpp
写数据函数CreateDataWriter增加写稀疏镜像分支
case WRITE_SPARSE:
{
std::unique_ptr<SparseWriter> writer(std::make_unique<SparseWriter>(partitionName));
return std::move(writer);
}
● 增加稀疏镜像类声明
.\base\update\updater\services\applypatch\raw_writer.h
增加稀疏镜像类声明及相关变量定义
typedef struct sparse_header {
uint32_t magic; /* 0xed26ff3a */
uint16_t major_version; /* (0x1) - reject images with higher major versions */
uint16_t minor_version; /* (0x0) - allow images with higer minor versions */
uint16_t file_hdr_sz; /* 28 bytes for first revision of the file format */
uint16_t chunk_hdr_sz; /* 12 bytes for first revision of the file format */
uint32_t blk_sz; /* block size in bytes, must be a multiple of 4 (4096) */
uint32_t total_blks; /* total blocks in the non-sparse output image */
uint32_t total_chunks; /* total chunks in the sparse input image */
uint32_t image_checksum; /* CRC32 checksum of the original data, counting "don't care" */
/* as 0. Standard 802.3 polynomial, use a Public Domain */
/* table implementation */
} sparse_header_t;
#define SPARSE_HEADER_MAGIC 0xed26ff3a
#define CHUNK_TYPE_RAW 0xCAC1
#define CHUNK_TYPE_FILL 0xCAC2
#define CHUNK_TYPE_DONT_CARE 0xCAC3
#define CHUNK_TYPE_CRC32 0xCAC4
typedef struct chunk_header {
uint16_t chunk_type; /* 0xCAC1 -> raw; 0xCAC2 -> fill; 0xCAC3 -> don't care */
uint16_t reserved1;
uint32_t chunk_sz; /* in blocks in output image */
uint32_t total_sz; /* in bytes of chunk input file including chunk header and data */
} chunk_header_t;
class SparseWriter : public DataWriter {
public:
virtual bool Write(const uint8_t *addr, size_t len, WriteMode mode, const std::string &partitionName);
explicit SparseWriter(const std::string partitionName) : offset_(0), fd_(-1), partitionName_(partitionName) {}
virtual ~SparseWriter()
{
offset_ = 0;
if (fd_ > 0) {
fsync(fd_);
close(fd_);
}
fd_ = -1;
}
private:
int WriteInternal(int fd, const uint8_t *data, size_t len, const std::string &partitionName);
SparseWriter(const SparseWriter&) = delete;
const SparseWriter& operator=(const SparseWriter&) = delete;
off64_t offset_;
int fd_;
std::string partitionName_;
};
● 增加稀疏镜像类实现
.\base\update\updater\services\applypatch\raw_writer.cpp
增加稀疏镜像类实现及相关变量定义，原有代码不变
bool SparseWriter::Write(const uint8_t *addr, size_t len, WriteMode mode, const std::string &partitionName)
{
if (addr == nullptr) {
LOG(ERROR) << "SparseWriter: invalid address.";
return false;
}
if (len == 0) {
LOG(INFO) << "SparseWriter: write length is 0, skip.";
return false;
}
if (fd_ < 0) {
fd_ = OpenPartition(partitionName_);
if (fd_ < 0) {
return false;
}
}
UPDATER_CHECK_ONLY_RETURN(WriteInternal(fd_, addr, len, partitionName_) >= 0, return false);
return true;
}
int SparseWriter::WriteInternal(int fd, const uint8_t *data, size_t len, const std::string &partitionName)
{
uint32_t written = 0;
sparse_header_t *sparse_header;
chunk_header_t *chunk_header;
unsigned int chunk;
void *membuf = NULL;
uint32_t *fill_buf = NULL;
uint32_t fill_val;
uint32_t bytes_written = 0;
uint32_t total_bytes = 0;
uint32_t blk = 0;
uint32_t chunk_data_sz = 0;
uint32_t blkcnt = 0;
uint32_t blks = 0;
uint32_t total_blocks = 0;
uint32_t addr_offset = 0;
uint32_t fill_buf_num_blks = 0;
uint32_t block_size = 4096;
uint32_t block_count = 524288;
uint32_t i;
uint32_t j;
int ret = lseek64(fd, offset_, SEEK_SET);
UPDATER_FILE_CHECK(ret != -1, "RawWriter: failed to seek file to " << offset_, return -1);
fill_buf_num_blks = CONFIG_FASTBOOT_FLASH_FILLBUF_SIZE / block_size;
LOG(INFO) << "WriteInternal offset_ " << offset_;
/* Read and skip over sparse image header */
sparse_header = (sparse_header_t *)data;
data += sparse_header->file_hdr_sz;
if (sparse_header->file_hdr_sz > sizeof(sparse_header_t)) {
/*
* Skip the remaining bytes in a header that is longer than
* we expected.
*/
data += (sparse_header->file_hdr_sz - sizeof(sparse_header_t));
}
LOG(INFO) << "=== Sparse Image Header ===";
LOG(INFO) << "magic: " << sparse_header->magic;
LOG(INFO) << "major_version: " << sparse_header->major_version;
LOG(INFO) << "minor_version: " << sparse_header->minor_version;
LOG(INFO) << "file_hdr_sz: " << sparse_header->file_hdr_sz;
LOG(INFO) << "chunk_hdr_sz: " << sparse_header->chunk_hdr_sz;
LOG(INFO) << "blk_sz: " << sparse_header->blk_sz;
LOG(INFO) << "total_blks: " << sparse_header->total_blks;
LOG(INFO) << "total_chunks: " << sparse_header->total_chunks;
LOG(INFO) << "Flashing Sparse Image";
blk = 0;
for (chunk = 0; chunk < sparse_header->total_chunks; chunk++) {
/* Read and skip over chunk header */
chunk_header = (chunk_header_t *)data;
data += sizeof(chunk_header_t);
if (chunk_header->chunk_type != CHUNK_TYPE_RAW)
{
LOG(INFO) << "=== Chunk Header ===";
LOG(INFO) << "chunk_type: " << chunk_header->chunk_type;
LOG(INFO) << "chunk_sz: " << chunk_header->chunk_sz;
LOG(INFO) << "total_sz: " << chunk_header->total_sz;
}
if (sparse_header->chunk_hdr_sz > sizeof(chunk_header_t)) {
/*
* Skip the remaining bytes in a header that is longer
* than we expected.
*/
data += (sparse_header->chunk_hdr_sz -
sizeof(chunk_header_t));
}
chunk_data_sz = sparse_header->blk_sz * chunk_header->chunk_sz;
blkcnt = chunk_data_sz / block_size;
switch (chunk_header->chunk_type) {
case CHUNK_TYPE_RAW:
if (chunk_header->total_sz !=
(sparse_header->chunk_hdr_sz + chunk_data_sz)) {
LOG(ERROR) << "Bogus chunk size for chunk type Raw";
return -1;
}
if (blk + blkcnt > 0 + block_count) {
LOG(ERROR) << "Request would exceed partition size!";
return -1;
}
addr_offset = blk * block_size;
ret = lseek64(fd, offset_ + addr_offset, SEEK_SET);
if (ret < 0) {
LOG(ERROR) << "failed to seek file to " << addr_offset << " error=" << strerror(errno);
return -1;
}
written = write(fd, data, blkcnt * block_size);
if (written < 0) {
LOG(ERROR) << "SparseWriter: failed to write data of len ";
return -1;
}
total_bytes = total_bytes + blkcnt * block_size;
blks = written / block_size;
blk += blks;
bytes_written += blkcnt * block_size;
total_blocks += chunk_header->chunk_sz;
data += chunk_data_sz;
break;
case CHUNK_TYPE_FILL:
if (chunk_header->total_sz !=
(sparse_header->chunk_hdr_sz + sizeof(uint32_t))) {
LOG(ERROR) << "Bogus chunk size for chunk type FILL total_sz err " << chunk_header->total_sz << "\n";
return -1;
}
ret = posix_memalign (&membuf, 64,
ROUNDUP(
block_size * fill_buf_num_blks,
64));
if (ret) {
LOG(ERROR) << "posix_memalign:" << strerror (errno);
return -1;
}
fill_buf = (uint32_t *)membuf;
if (!fill_buf) {
LOG(ERROR) << "Malloc failed for: CHUNK_TYPE_FILL";
return -1;
}
fill_val = *(uint32_t *)data;
data = data + sizeof(uint32_t);
for (i = 0;
i < (block_size * fill_buf_num_blks /
sizeof(fill_val));
i++)
fill_buf[i] = fill_val;
if (blk + blkcnt > 0 + block_count) {
LOG(ERROR) << "Request would exceed partition size!";
return -1;
}
for (i = 0; i < blkcnt;) {
j = blkcnt - i;
if (j > fill_buf_num_blks)
j = fill_buf_num_blks;
addr_offset = blk * block_size;
ret = lseek64(fd, offset_ + addr_offset, SEEK_SET);
if (ret < 0) {
LOG(ERROR) << "failed to lseek file to " << addr_offset << " error=" << strerror(errno);
return -1;
}
written = write(fd, fill_buf, j * block_size);
if (written < 0) {
LOG(ERROR) << "SparseWriter: failed to write data of len ";
return -1;
}
total_bytes = total_bytes + j * block_size;
blks = written / block_size;
if (blks < j) {
LOG(ERROR) << "Write failed, block";
free(fill_buf);
return -1;
}
blk += blks;
i += j;
}
bytes_written += blkcnt * block_size;
total_blocks += chunk_data_sz / sparse_header->blk_sz;
free(fill_buf);
break;
case CHUNK_TYPE_DONT_CARE:
blk += blkcnt;
total_blocks += chunk_header->chunk_sz;
break;
case CHUNK_TYPE_CRC32:
if (chunk_header->total_sz !=
sparse_header->chunk_hdr_sz) {
LOG(ERROR) << "Bogus chunk size for chunk type CRC32 total_sz err " << chunk_header->total_sz;
return -1;
}
total_blocks += chunk_header->chunk_sz;
data += chunk_data_sz;
break;
default:
LOG(INFO) << __func__ << ": Unknown chunk type: " << chunk_header->chunk_type;
return -1;
}
}
LOG(INFO) << "Wrote "<< chunk <<"blocks, expected to write " << sparse_header->total_blks << "blocks\n";
LOG(INFO) << "........ wrote "<< bytes_written <<"bytes to " << partitionName << "\n";
LOG(INFO) << "total_bytes=" << total_bytes;
return 0;
}3、验证稀疏镜像升级
（1）拷贝升级包
将制作的稀疏镜像升级包通过HDC工具推进系统data目录，并修改文件名为updater.zip，路径如下：/data/updater/updater.zip
（2）启动升级
系统启动后，在命令行工具输入如下命令启动升级
reboot updater:--update_package=/data/updater/updater.zip
输入命令后，系统重启，进入升级页面，等待升级完成。
本文介绍了OpenHarmony系统中实现稀疏镜像升级的方法，理解稀疏镜像原理及稀疏镜像还原方法可以快速在自己的系统中应用稀疏镜像升级，提高系统升级速度。