MongoDB杂谈

1
2
3
4

timestamp(big endian)  machine hash    pid        counter
|<----------------->|<------------>|<------->|<------------>|
[----|----|----|----|----|----|----|----|----|----|----|----]
0                   4                   8                   12

1
2
3
4

timestamp(big endian)   process unique data      counter
|<----------------->|<---------------------->|<------------>|
[----|----|----|----|----|----|----|----|----|----|----|----]
0                   4                   8                   12

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

/*C版本 https://github.com/mongodb/mongo-c-driver/blob/454a01422ee61d2add82f054aa3799750b7947e2/src/libbson/src/bson/bson-context.c#L268
* C版本将时间、进程id、主机名称异或得到一个随机种子，再随机5字节机器码和counter初始值。
*/
static void
_bson_context_init_random (bson_context_t *context, bool init_sequence)
{
   int64_t rand_bytes;
   struct timeval tv;
   unsigned int seed = 0;
   char hostname[HOST_NAME_MAX];
   char *ptr;
   int hostname_chars_left;

   /*
    * The seed consists of the following xor'd together:
    * - current time in seconds
    * - current time in milliseconds
    * - current pid
    * - current hostname
    */
   bson_gettimeofday (&tv);
   seed ^= (unsigned int) tv.tv_sec;
   seed ^= (unsigned int) tv.tv_usec;
   seed ^= (unsigned int) context->pid;

   context->gethostname (hostname);
   hostname_chars_left = strlen (hostname);
   ptr = hostname;
   while (hostname_chars_left) {
      uint32_t hostname_chunk = 0;
      uint32_t to_copy = hostname_chars_left > 4 ? 4 : hostname_chars_left;

      memcpy (&hostname_chunk, ptr, to_copy);
      seed ^= (unsigned int) hostname_chunk;
      hostname_chars_left -= to_copy;
      ptr += to_copy;
   }

#ifndef BSON_HAVE_RAND_R
   srand (seed);
#endif

   /* Generate a seed for the random starting position of our increment
    * bytes and the five byte random number. */
   if (init_sequence) {
      /* We mask off the last nibble so that the last digit of the OID will
       * start at zero. Just to be nice. */
      context->seq32 = _get_rand (&seed) & 0x007FFFF0;
   }

   rand_bytes = _get_rand (&seed);
   rand_bytes <<= 32;
   rand_bytes |= _get_rand (&seed);

   /* Copy five random bytes, endianness does not matter. */
   memcpy (&context->rand, (char *) &rand_bytes, sizeof (context->rand));
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

/* golang版本 https://github.com/mongodb/mongo-go-driver/blob/master/bson/primitive/objectid.go
* go版本使用 rand.Reader 直接产生随机数，rand.Reader 是一个密码学安全的随机数生成器的公共实例
* rand.Reader在Linux下使用/dev/urandom实现，在windows下使用 RtlGenRandom 实现
*/
...
var objectIDCounter = readRandomUint32()
var processUnique = processUniqueBytes()
...
func processUniqueBytes() [5]byte {
	var b [5]byte
	_, err := io.ReadFull(rand.Reader, b[:])
	if err != nil {
		panic(fmt.Errorf("cannot initialize objectid package with crypto.rand.Reader: %v", err))
	}

	return b
}

func readRandomUint32() uint32 {
	var b [4]byte
	_, err := io.ReadFull(rand.Reader, b[:])
	if err != nil {
		panic(fmt.Errorf("cannot initialize objectid package with crypto.rand.Reader: %v", err))
	}

	return (uint32(b[0]) << 0) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24)
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

# python版本 https://github.com/mongodb/mongo-python-driver/blob/master/bson/objectid.py
# python 使用 os.urandom 生成随机数, os.urandom 跟 go 的类似，使用/dev/urandom 或者 系统提供的 api
# os.urandom() method is used to generate a string of size random bytes suitable for cryptographic use or we can say this method generates a string containing random characters.
...
def _random_bytes():
    """Get the 5-byte random field of an ObjectId."""
    return os.urandom(5)


class ObjectId(object):
    """A MongoDB ObjectId.
    """

    _pid = os.getpid()

    _inc = SystemRandom().randint(0, _MAX_COUNTER_VALUE)
    _inc_lock = threading.Lock()

    __random = _random_bytes()
...

1
2

# 创建TTL索引
db.yourdb.createIndex( { "fieldKey": 1 }, { expireAfterSeconds: 3600 } )

1
2
3
4
5

# 伪代码
db.yourdb.createIndex( { "fieldKey": 1 }, { expireAfterSeconds: 1 } )
db.yourdb.insert({"fieldKey": "hehe"})
sleep(5)
assert(db.yourdb.findOne({"fieldKey": "hehe"}) == null)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

// https://github.com/mongodb/mongo/blob/master/src/mongo/db/ttl.cpp
...
class TTLMonitor : public BackgroundJob {
    void run() {
        {
            // Wait until either ttlMonitorSleepSecs passes or a shutdown is requested.
            auto deadline = Date_t::now() + Seconds(ttlMonitorSleepSecs.load());
            stdx::unique_lock<Latch> lk(_stateMutex);

            MONGO_IDLE_THREAD_BLOCK;
            _shuttingDownCV.wait_until(
                    lk, deadline.toSystemTimePoint(), [&] { return _shuttingDown; });

            if (_shuttingDown) {
                return;
            }
        }
        ...
        doTTLPass();
        ...
    }
    
    /**
     * Gets all TTL specifications for every collection and deletes expired documents.
     */
    void doTTLPass() {
        ...
    }
}
...

1
2
3
4
5
6

struct MsgHeader {
    int32   messageLength; // 消息的总长度，包括这个字段本身
    int32   requestID;     // 请求的标识id，同一个客户端，每次请求都生成一个唯一id
    int32   responseTo;    // 原始的requestID，回复给客户端，客户端用于处理请求回复
    int32   opCode;        // 请求的类型，不同的类型对应不同的消息体
}

1
2
3
4
5
6

OP_MSG {
    MsgHeader header;          // standard message header
    uint32 flagBits;           // message flags
    Sections[] sections;       // data sections
    optional<uint32> checksum; // optional CRC-32C checksum
}