海量数据处理利器 Roaring BitMap 原理介绍

public void add(final int x) {
    final short hb = Util.highbits(x);
    final int i = highLowContainer.getIndex(hb);
    if (i >= 0) {
      highLowContainer.setContainerAtIndex(i,
          highLowContainer.getContainerAtIndex(i).add(Util.lowbits(x)));
    } else {
      final ArrayContainer newac = new ArrayContainer();
      highLowContainer.insertNewKeyValueAt(-i - 1, hb, newac.add(Util.lowbits(x)));
    }
  }

// 包含一个二分查找
protected int getIndex(short x) {
  // 在二分查找之前，我们先对常见情况优化。
  if ((size == 0) || (keys[size - 1] == x)) {
    return size - 1;
  }
  // 没有碰到常见情况，我们只能遍历这个列表。
  return this.binarySearch(0, size, x);
}

if (cardinality == 0 || (cardinality > 0
          && toIntUnsigned(x) > toIntUnsigned(content[cardinality - 1]))) {
    if (cardinality >= DEFAULT_MAX_SIZE) {
      return toBitMapContainer().add(x);
    }
    if (cardinality >= this.content.length) {
      increaseCapacity();
    }
    content[cardinality++] = x;
  } else {
    int loc = Util.unsignedBinarySearch(content, 0, cardinality, x);
    if (loc < 0) {
      // 当标签中元素数量等于默认最大值时，把ArrayContainer转换为BitMapContainer
      if (cardinality >= DEFAULT_MAX_SIZE) {
        return toBitMapContainer().add(x);
      }
      if (cardinality >= this.content.length) {
        increaseCapacity();
      }
      System.arraycopy(content, -loc - 1, content, -loc, cardinality + loc + 1);
      content[-loc - 1] = x;
      ++cardinality;
    }
  }
  return this;
}

public Container add(final short i) {
  final int x = Util.toIntUnsigned(i);
  final long previous = BitMap[x / 64];
  long newval = previous | (1L << x);   BitMap[x / 64] = newval;
  if (USE_BRANCHLESS) {
    cardinality += (previous ^ newval) >>> x;
  } else if (previous != newval) {
    ++cardinality;
  }
  return this;
}

public Container add(short k) {
   
  int index = unsignedInterleavedBinarySearch(valueslength, 0, nbrruns, k);
  if (index >= 0) {
    return this;// already there
  }
  index = -index - 2;
  if (index >= 0) {
    int offset = toIntUnsigned(k) - toIntUnsigned(getValue(index));
    int le = toIntUnsigned(getLength(index));
    if (offset <= le) {
      return this;
    }
    if (offset == le + 1) {
      // we may need to fuse
      if (index + 1 < nbrruns) {
        if (toIntUnsigned(getValue(index + 1)) == toIntUnsigned(k) + 1) {
          // indeed fusion is needed
          setLength(index,
              (short) (getValue(index + 1) + getLength(index + 1) - getValue(index)));
          recoverRoomAtIndex(index + 1);
          return this;
        }
      }
      incrementLength(index);
      return this;
    }
    if (index + 1 < nbrruns) {
      // we may need to fuse
      if (toIntUnsigned(getValue(index + 1)) == toIntUnsigned(k) + 1) {
        // indeed fusion is needed
        setValue(index + 1, k);
        setLength(index + 1, (short) (getLength(index + 1) + 1));
        return this;
      }
    }
  }
  if (index == -1) {
    // we may need to extend the first run
    if (0 < nbrruns) {
      if (getValue(0) == k + 1) {
        incrementLength(0);
        decrementValue(0);
        return this;
      }
    }
  }
  makeRoomAtIndex(index + 1);
  setValue(index + 1, k);
  setLength(index + 1, (short) 0);
  return this;
}

public static void count(Integer inputSize) {         RoaringBitMap BitMap = new RoaringBitMap();         BitMap.add(0L, inputSize);
 
        //获取BitMap个数
        int cardinality = BitMap.getCardinality();
 
        //获取BitMap压缩大小
        int compressSizeIntBytes = BitMap.getSizeInBytes();
 
        //删除压缩（移除行程编码，将container退化为BitMapContainer 或 ArrayContainer）         BitMap.removeRunCompression();
 
        //获取BitMap不压缩大小
        int uncompressSizeIntBytes = BitMap.getSizeInBytes();
 
        System.out.println("Roaring BitMap个数：" + cardinality);
        System.out.println("最好情况，BitMap压缩大小：" + compressSizeIntBytes / 1024 + "KB");
        System.out.println("最坏情况，BitMap不压缩大小：" + uncompressSizeIntBytes / 1024 / 1024 + "MB");
 
        BitSet bitSet = new BitSet();
        for (int i = 0; i < inputSize; i++) {
            bitSet.set(i);
        }
        //获取BitMap大小
        int size = bitSet.size();
 
        System.out.println("BitMap个数：" + bitSet.length());
        System.out.println("BitMap大小：" + size / 8 / 1024 / 1024 + "MB");
    }

//添加单个数字
public void add(final int x)


//添加范围数字
public void add(final long rangeStart, final long rangeEnd)


//移除数字
public void remove(final int x)


//遍历RBM
public void forEach(IntConsumer ic)


//检测是否包含
public boolean contains(final int x)


//获取基数
public int getCardinality()


//位与，取两个RBM的交集，当前RBM会被修改
public void and(final RoaringBitMap x2)


//同上，但是会返回一个新的RBM，不会修改原始的RBM，线程安全
public static RoaringBitMap and(final RoaringBitMap x1, final RoaringBitMap x2)


//位或，取两个RBM的并集，当前RBM会被修改
public void or(final RoaringBitMap x2)


//同上，但是会返回一个新的RBM，不会修改原始的RBM，线程安全
public static RoaringBitMap or(final RoaringBitMap x1, final RoaringBitMap x2)


//异或，取两个RBM的对称差，当前RBM会被修改
public void xor(final RoaringBitMap x2)


//同上，但是会返回一个新的RBM，不会修改原始的RBM，线程安全
public static RoaringBitMap xor(final RoaringBitMap x1, final RoaringBitMap x2)


//取原始值和x2的差集，当前RBM会被修改
public void andNot(final RoaringBitMap x2)


//同上，但是会返回一个新的RBM，不会修改原始的RBM，线程安全
public static RoaringBitMap andNot(final RoaringBitMap x1, final RoaringBitMap x2)


//序列化
public void serialize(DataOutput out) throws IOException
public void serialize(ByteBuffer buffer)


//反序列化
public void deserialize(DataInput in) throws IOException
public void deserialize(ByteBuffer bbf) throws IOException