- 首页 > it技术 > >
/*
* Copyright (C) 2012 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/// This is an on-disk cache which maps a 64-bits key to a byte array.
//
// It consists of three files: one index file and two data files. One of the
// data files is "active", and the other is "inactive". New entries are
// appended into the active region until it reaches the size limit. At that
// point the active file and the inactive file are swapped, and the new active
// file is truncated to empty (and the index for that file is also cleared).
// The index is a hash table with linear probing. When the load factor reaches
// 0.5, it does the same thing like when the size limit is reached.
//
// The index file format: (all numbers are stored in little-endian)
// [0]Magic number: 0xB3273030
// [4]MaxEntries: Max number of hash entries per region.
// [8]MaxBytes: Max number of data bytes per region (including header).
// [12] ActiveRegion: The active growing region: 0 or 1.
// [16] ActiveEntries: The number of hash entries used in the active region.
// [20] ActiveBytes: The number of data bytes used in the active region.
// [24] Version number.
// [28] Checksum of [0..28).
// [32] Hash entries for region 0. The size is X = (12 * MaxEntries bytes).
// [32 + X] Hash entries for region 1. The size is also X.
//
// Each hash entry is 12 bytes: 8 bytes key and 4 bytes offset into the data
// file. The offset is 0 when the slot is free. Note that 0 is a valid value
// for key. The keys are used directly as index into a hash table, so they
// should be suitably distributed.
//
// Each data file stores data for one region. The data file is concatenated
// blobs followed by the magic number 0xBD248510.
//
// The blob format:
// [0]Key of this blob
// [8]Checksum of this blob
// [12] Offset of this blob
// [16] Length of this blob (not including header)
// [20] Blob
//
// Below are the interface for BlobCache. The instance of this class does not
// support concurrent use by multiple threads.
//
// public BlobCache(String path, int maxEntries, int maxBytes, boolean reset) throws IOException;
// public void insert(long key, byte[] data) throws IOException;
// public byte[] lookup(long key) throws IOException;
// public void lookup(LookupRequest req) throws IOException;
// public void close();
// public void syncIndex();
// public void syncAll();
// public static void deleteFiles(String path);
//
package com.android.mms.util;
import android.util.Log;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.zip.Adler32;
public class BlobCache implements Closeable {
private static final String TAG = "BlobCache";
private static final int MAGIC_INDEX_FILE = 0xB3273030;
private static final int MAGIC_DATA_FILE = 0xBD248510;
// index header offset
private static final int IH_MAGIC = 0;
private static final int IH_MAX_ENTRIES = 4;
private static final int IH_MAX_BYTES = 8;
private static final int IH_ACTIVE_REGION = 12;
private static final int IH_ACTIVE_ENTRIES = 16;
private static final int IH_ACTIVE_BYTES = 20;
private static final int IH_VERSION = 24;
private static final int IH_CHECKSUM = 28;
private static final int INDEX_HEADER_SIZE = 32;
private static final int DATA_HEADER_SIZE = 4;
// blob header offset
private static final int BH_KEY = 0;
private static final int BH_CHECKSUM = 8;
private static final int BH_OFFSET = 12;
private static final int BH_LENGTH = 16;
private static final int BLOB_HEADER_SIZE = 20;
private RandomAccessFile mIndexFile;
private RandomAccessFile mDataFile0;
private RandomAccessFile mDataFile1;
private FileChannel mIndexChannel;
private MappedByteBuffer mIndexBuffer;
private int mMaxEntries;
private int mMaxBytes;
private int mActiveRegion;
private int mActiveEntries;
private int mActiveBytes;
private int mVersion;
private RandomAccessFile mActiveDataFile;
private RandomAccessFile mInactiveDataFile;
private int mActiveHashStart;
private int mInactiveHashStart;
private byte[] mIndexHeader = new byte[INDEX_HEADER_SIZE];
private byte[] mBlobHeader = new byte[BLOB_HEADER_SIZE];
private Adler32 mAdler32 = new Adler32();
// Creates the cache. Three files will be created:
// path + ".idx", path + ".0", and path + ".1"
// The ".0" file and the ".1" file each stores data for a region. Each of
// them can grow to the size specified by maxBytes. The maxEntries parameter
// specifies the maximum number of entries each region can have. If the
// "reset" parameter is true, the cache will be cleared before use.
public BlobCache(String path, int maxEntries, int maxBytes, boolean reset)
throws IOException {
this(path, maxEntries, maxBytes, reset, 0);
}public BlobCache(String path, int maxEntries, int maxBytes, boolean reset,
int version) throws IOException {
mIndexFile = new RandomAccessFile(path + ".idx", "rw");
mDataFile0 = new RandomAccessFile(path + ".0", "rw");
mDataFile1 = new RandomAccessFile(path + ".1", "rw");
mVersion = version;
if (!reset && loadIndex()) {
return;
}resetCache(maxEntries, maxBytes);
if (!loadIndex()) {
closeAll();
throw new IOException("unable to load index");
}
}// Delete the files associated with the given path previously created
// by the BlobCache constructor.
public static void deleteFiles(String path) {
deleteFileSilently(path + ".idx");
deleteFileSilently(path + ".0");
deleteFileSilently(path + ".1");
}private static void deleteFileSilently(String path) {
try {
new File(path).delete();
} catch (Throwable t) {
// ignore;
}
}// Close the cache. All resources are released. No other method should be
// called after this is called.
@Override
public void close() {
syncAll();
closeAll();
}private void closeAll() {
closeSilently(mIndexChannel);
closeSilently(mIndexFile);
closeSilently(mDataFile0);
closeSilently(mDataFile1);
}// Returns true if loading index is successful. After this method is called,
// mIndexHeader and index header in file should be kept sync.
private boolean loadIndex() {
try {
mIndexFile.seek(0);
mDataFile0.seek(0);
mDataFile1.seek(0);
byte[] buf = mIndexHeader;
if (mIndexFile.read(buf) != INDEX_HEADER_SIZE) {
Log.w(TAG, "cannot read header");
return false;
}if (readInt(buf, IH_MAGIC) != MAGIC_INDEX_FILE) {
Log.w(TAG, "cannot read header magic");
return false;
}if (readInt(buf, IH_VERSION) != mVersion) {
Log.w(TAG, "version mismatch");
return false;
}mMaxEntries = readInt(buf, IH_MAX_ENTRIES);
mMaxBytes = readInt(buf, IH_MAX_BYTES);
mActiveRegion = readInt(buf, IH_ACTIVE_REGION);
mActiveEntries = readInt(buf, IH_ACTIVE_ENTRIES);
mActiveBytes = readInt(buf, IH_ACTIVE_BYTES);
int sum = readInt(buf, IH_CHECKSUM);
if (checkSum(buf, 0, IH_CHECKSUM) != sum) {
Log.w(TAG, "header checksum does not match");
return false;
}// Sanity check
if (mMaxEntries <= 0) {
Log.w(TAG, "invalid max entries");
return false;
}
if (mMaxBytes <= 0) {
Log.w(TAG, "invalid max bytes");
return false;
}
if (mActiveRegion != 0 && mActiveRegion != 1) {
Log.w(TAG, "invalid active region");
return false;
}
if (mActiveEntries < 0 || mActiveEntries > mMaxEntries) {
Log.w(TAG, "invalid active entries");
return false;
}
if (mActiveBytes < DATA_HEADER_SIZE || mActiveBytes > mMaxBytes) {
Log.w(TAG, "invalid active bytes");
return false;
}
if (mIndexFile.length() !=
INDEX_HEADER_SIZE + mMaxEntries * 12 * 2) {
Log.w(TAG, "invalid index file length");
return false;
}// Make sure data file has magic
byte[] magic = new byte[4];
if (mDataFile0.read(magic) != 4) {
Log.w(TAG, "cannot read data file magic");
return false;
}
if (readInt(magic, 0) != MAGIC_DATA_FILE) {
Log.w(TAG, "invalid data file magic");
return false;
}
if (mDataFile1.read(magic) != 4) {
Log.w(TAG, "cannot read data file magic");
return false;
}
if (readInt(magic, 0) != MAGIC_DATA_FILE) {
Log.w(TAG, "invalid data file magic");
return false;
}// Map index file to memory
mIndexChannel = mIndexFile.getChannel();
mIndexBuffer = mIndexChannel.map(FileChannel.MapMode.READ_WRITE,
0, mIndexFile.length());
mIndexBuffer.order(ByteOrder.LITTLE_ENDIAN);
setActiveVariables();
return true;
} catch (IOException ex) {
Log.e(TAG, "loadIndex failed.", ex);
return false;
}
}private void setActiveVariables() throws IOException {
mActiveDataFile = (mActiveRegion == 0) ? mDataFile0 : mDataFile1;
mInactiveDataFile = (mActiveRegion == 1) ? mDataFile0 : mDataFile1;
mActiveDataFile.setLength(mActiveBytes);
mActiveDataFile.seek(mActiveBytes);
mActiveHashStart = INDEX_HEADER_SIZE;
mInactiveHashStart = INDEX_HEADER_SIZE;
if (mActiveRegion == 0) {
mInactiveHashStart += mMaxEntries * 12;
} else {
mActiveHashStart += mMaxEntries * 12;
}
}private void resetCache(int maxEntries, int maxBytes) throws IOException {
mIndexFile.setLength(0);
// truncate to zero the index
mIndexFile.setLength(INDEX_HEADER_SIZE + maxEntries * 12 * 2);
mIndexFile.seek(0);
byte[] buf = mIndexHeader;
writeInt(buf, IH_MAGIC, MAGIC_INDEX_FILE);
writeInt(buf, IH_MAX_ENTRIES, maxEntries);
writeInt(buf, IH_MAX_BYTES, maxBytes);
writeInt(buf, IH_ACTIVE_REGION, 0);
writeInt(buf, IH_ACTIVE_ENTRIES, 0);
writeInt(buf, IH_ACTIVE_BYTES, DATA_HEADER_SIZE);
writeInt(buf, IH_VERSION, mVersion);
writeInt(buf, IH_CHECKSUM, checkSum(buf, 0, IH_CHECKSUM));
mIndexFile.write(buf);
// This is only needed if setLength does not zero the extended part.
// writeZero(mIndexFile, maxEntries * 12 * 2);
mDataFile0.setLength(0);
mDataFile1.setLength(0);
mDataFile0.seek(0);
mDataFile1.seek(0);
writeInt(buf, 0, MAGIC_DATA_FILE);
mDataFile0.write(buf, 0, 4);
mDataFile1.write(buf, 0, 4);
}// Flip the active region and the inactive region.
private void flipRegion() throws IOException {
mActiveRegion = 1 - mActiveRegion;
mActiveEntries = 0;
mActiveBytes = DATA_HEADER_SIZE;
writeInt(mIndexHeader, IH_ACTIVE_REGION, mActiveRegion);
writeInt(mIndexHeader, IH_ACTIVE_ENTRIES, mActiveEntries);
writeInt(mIndexHeader, IH_ACTIVE_BYTES, mActiveBytes);
updateIndexHeader();
setActiveVariables();
clearHash(mActiveHashStart);
syncIndex();
}// Sync mIndexHeader to the index file.
private void updateIndexHeader() {
writeInt(mIndexHeader, IH_CHECKSUM,
checkSum(mIndexHeader, 0, IH_CHECKSUM));
mIndexBuffer.position(0);
mIndexBuffer.put(mIndexHeader);
}// Clear the hash table starting from the specified offset.
private void clearHash(int hashStart) {
byte[] zero = new byte[1024];
mIndexBuffer.position(hashStart);
for (int count = mMaxEntries * 12;
count > 0;
) {
int todo = Math.min(count, 1024);
mIndexBuffer.put(zero, 0, todo);
count -= todo;
}
}// Inserts a (key, data) pair into the cache.
public void insert(long key, byte[] data) throws IOException {
if (DATA_HEADER_SIZE + BLOB_HEADER_SIZE + data.length > mMaxBytes) {
throw new RuntimeException("blob is too large!");
}if (mActiveBytes + BLOB_HEADER_SIZE + data.length > mMaxBytes
|| mActiveEntries * 2 >= mMaxEntries) {
flipRegion();
}if (!lookupInternal(key, mActiveHashStart)) {
// If we don't have an existing entry with the same key, increase
// the entry count.
mActiveEntries++;
writeInt(mIndexHeader, IH_ACTIVE_ENTRIES, mActiveEntries);
}insertInternal(key, data, data.length);
updateIndexHeader();
}// Appends the data to the active file. It also updates the hash entry.
// The proper hash entry (suitable for insertion or replacement) must be
// pointed by mSlotOffset.
private void insertInternal(long key, byte[] data, int length)
throws IOException {
byte[] header = mBlobHeader;
int sum = checkSum(data);
writeLong(header, BH_KEY, key);
writeInt(header, BH_CHECKSUM, sum);
writeInt(header, BH_OFFSET, mActiveBytes);
writeInt(header, BH_LENGTH, length);
mActiveDataFile.write(header);
mActiveDataFile.write(data, 0, length);
mIndexBuffer.putLong(mSlotOffset, key);
mIndexBuffer.putInt(mSlotOffset + 8, mActiveBytes);
mActiveBytes += BLOB_HEADER_SIZE + length;
writeInt(mIndexHeader, IH_ACTIVE_BYTES, mActiveBytes);
}public static class LookupRequest {
public long key;
// input: the key to find
public byte[] buffer;
// input/output: the buffer to store the blob
public int length;
// output: the length of the blob
}// This method is for one-off lookup. For repeated lookup, use the version
// accepting LookupRequest to avoid repeated memory allocation.
private LookupRequest mLookupRequest = new LookupRequest();
public byte[] lookup(long key) throws IOException {
mLookupRequest.key = key;
mLookupRequest.buffer = null;
if (lookup(mLookupRequest)) {
return mLookupRequest.buffer;
} else {
return null;
}
}// Returns true if the associated blob for the given key is available.
// The blob is stored in the buffer pointed by req.buffer, and the length
// is in stored in the req.length variable.
//
// The user can input a non-null value in req.buffer, and this method will
// try to use that buffer. If that buffer is not large enough, this method
// will allocate a new buffer and assign it to req.buffer.
//
// This method tries not to throw IOException even if the data file is
// corrupted, but it can still throw IOException if things get strange.
public boolean lookup(LookupRequest req) throws IOException {
// Look up in the active region first.
if (lookupInternal(req.key, mActiveHashStart)) {
if (getBlob(mActiveDataFile, mFileOffset, req)) {
return true;
}
}// We want to copy the data from the inactive file to the active file
// if it's available. So we keep the offset of the hash entry so we can
// avoid looking it up again.
int insertOffset = mSlotOffset;
// Look up in the inactive region.
if (lookupInternal(req.key, mInactiveHashStart)) {
if (getBlob(mInactiveDataFile, mFileOffset, req)) {
// If we don't have enough space to insert this blob into
// the active file, just return it.
if (mActiveBytes + BLOB_HEADER_SIZE + req.length > mMaxBytes
|| mActiveEntries * 2 >= mMaxEntries) {
return true;
}
// Otherwise copy it over.
mSlotOffset = insertOffset;
try {
insertInternal(req.key, req.buffer, req.length);
mActiveEntries++;
writeInt(mIndexHeader, IH_ACTIVE_ENTRIES, mActiveEntries);
updateIndexHeader();
} catch (Throwable t) {
Log.e(TAG, "cannot copy over");
}
return true;
}
}return false;
}// Copies the blob for the specified offset in the specified file to
// req.buffer. If req.buffer is null or too small, allocate a buffer and
// assign it to req.buffer.
// Returns false if the blob is not available (either the index file is
// not sync with the data file, or one of them is corrupted). The length
// of the blob is stored in the req.length variable.
private boolean getBlob(RandomAccessFile file, int offset,
LookupRequest req) throws IOException {
byte[] header = mBlobHeader;
long oldPosition = file.getFilePointer();
try {
file.seek(offset);
if (file.read(header) != BLOB_HEADER_SIZE) {
Log.w(TAG, "cannot read blob header");
return false;
}
long blobKey = readLong(header, BH_KEY);
if (blobKey != req.key) {
Log.w(TAG, "blob key does not match: " + blobKey);
return false;
}
int sum = readInt(header, BH_CHECKSUM);
int blobOffset = readInt(header, BH_OFFSET);
if (blobOffset != offset) {
Log.w(TAG, "blob offset does not match: " + blobOffset);
return false;
}
int length = readInt(header, BH_LENGTH);
if (length < 0 || length > mMaxBytes - offset - BLOB_HEADER_SIZE) {
Log.w(TAG, "invalid blob length: " + length);
return false;
}
if (req.buffer == null || req.buffer.length < length) {
req.buffer = new byte[length];
}byte[] blob = req.buffer;
req.length = length;
if (file.read(blob, 0, length) != length) {
Log.w(TAG, "cannot read blob data");
return false;
}
if (checkSum(blob, 0, length) != sum) {
Log.w(TAG, "blob checksum does not match: " + sum);
return false;
}
return true;
} catch (Throwable t){
Log.e(TAG, "getBlob failed.", t);
return false;
} finally {
file.seek(oldPosition);
}
}// Tries to look up a key in the specified hash region.
// Returns true if the lookup is successful.
// The slot offset in the index file is saved in mSlotOffset. If the lookup
// is successful, it's the slot found. Otherwise it's the slot suitable for
// insertion.
// If the lookup is successful, the file offset is also saved in
// mFileOffset.
private int mSlotOffset;
private int mFileOffset;
private boolean lookupInternal(long key, int hashStart) {
int slot = (int) (key % mMaxEntries);
if (slot < 0) slot += mMaxEntries;
int slotBegin = slot;
while (true) {
int offset = hashStart + slot * 12;
long candidateKey = mIndexBuffer.getLong(offset);
int candidateOffset = mIndexBuffer.getInt(offset + 8);
if (candidateOffset == 0) {
mSlotOffset = offset;
return false;
} else if (candidateKey == key) {
mSlotOffset = offset;
mFileOffset = candidateOffset;
return true;
} else {
if (++slot >= mMaxEntries) {
slot = 0;
}
if (slot == slotBegin) {
Log.w(TAG, "corrupted index: clear the slot.");
mIndexBuffer.putInt(hashStart + slot * 12 + 8, 0);
}
}
}
}public void syncIndex() {
try {
mIndexBuffer.force();
} catch (Throwable t) {
Log.w(TAG, "sync index failed", t);
}
}public void syncAll() {
syncIndex();
try {
mDataFile0.getFD().sync();
} catch (Throwable t) {
Log.w(TAG, "sync data file 0 failed", t);
}
try {
mDataFile1.getFD().sync();
} catch (Throwable t) {
Log.w(TAG, "sync data file 1 failed", t);
}
}// This is for testing only.
//
// Returns the active count (mActiveEntries). This also verifies that
// the active count matches matches what's inside the hash region.
int getActiveCount() {
int count = 0;
for (int i = 0;
i < mMaxEntries;
i++) {
int offset = mActiveHashStart + i * 12;
long candidateKey = mIndexBuffer.getLong(offset);
int candidateOffset = mIndexBuffer.getInt(offset + 8);
if (candidateOffset != 0) ++count;
}
if (count == mActiveEntries) {
return count;
} else {
Log.e(TAG, "wrong active count: " + mActiveEntries + " vs " + count);
return -1;
// signal failure.
}
}int checkSum(byte[] data) {
mAdler32.reset();
mAdler32.update(data);
return (int) mAdler32.getValue();
}int checkSum(byte[] data, int offset, int nbytes) {
mAdler32.reset();
mAdler32.update(data, offset, nbytes);
return (int) mAdler32.getValue();
}static void closeSilently(Closeable c) {
if (c == null) return;
try {
c.close();
} catch (Throwable t) {
// do nothing
}
}static int readInt(byte[] buf, int offset) {
return (buf[offset] & 0xff)
| ((buf[offset + 1] & 0xff) << 8)
| ((buf[offset + 2] & 0xff) << 16)
| ((buf[offset + 3] & 0xff) << 24);
}static long readLong(byte[] buf, int offset) {
long result = buf[offset + 7] & 0xff;
for (int i = 6;
i >= 0;
i--) {
result = (result << 8) | (buf[offset + i] & 0xff);
}
return result;
}static void writeInt(byte[] buf, int offset, int value) {
for (int i = 0;
i < 4;
i++) {
buf[offset + i] = (byte) (value & 0xff);
value >>= 8;
}
}static void writeLong(byte[] buf, int offset, long value) {
for (int i = 0;
i < 8;
i++) {
buf[offset + i] = (byte) (value & 0xff);
value >>= 8;
}
}
}
推荐阅读
