Some notes:

1. both masker and maskee need to set cacheAsBitmap to true
2. have to use transparent png or transparent swf file for masker, no jpg
3. call setMask after loadMovie finished
4. maskee can be multiple layers. Only need to setMask on top layer.

Sample code:

/*this.createEmptyMovieClip("mask_mc", 2);
mask_mc.createEmptyMovieClip("mask_submc", mask_mc.getNextHighestDepth());
mask_mc.cacheAsBitmap = true;

mask_mc.mask_submc.beginFill(0xfff000);
mask_mc.mask_submc.moveTo(0, 100);
mask_mc.mask_submc.lineTo(200, 100);
mask_mc.mask_submc.lineTo(200, 300);
mask_mc.mask_submc.lineTo(0, 300);
mask_mc.mask_submc.lineTo(0, 100);
mask_mc.mask_submc.endFill();*/


attachMovie("masker","mask_mc", 2);
mask_mc.cacheAsBitmap = true;

this.createEmptyMovieClip("mcCharMask", 1) 
mcCharMask.loadMovie("a.png");

this.onEnterFrame = function(){
		if(mcCharMask.getBytesLoaded() >= mcCharMask.getBytesTotal() && mcCharMask.getBytesLoaded() > 0){
		delete this.onEnterFrame;
		// set cacheAsBitmap before setMask. Sometime cacheAsBitmap value changes after loadMovie
		mcCharMask.cacheAsBitmap = true;
		mask_mc.setMask(mcCharMask);
		
		/*mask_mc.createEmptyMovieClip("mask_submc1", mask_mc.getNextHighestDepth());

		mask_mc.mask_submc1.beginFill(0xff0000);
mask_mc.mask_submc1.moveTo(0, 200);
mask_mc.mask_submc1.lineTo(200, 200);
mask_mc.mask_submc1.lineTo(200, 300);
mask_mc.mask_submc1.lineTo(0, 300);
mask_mc.mask_submc1.lineTo(0, 200);
mask_mc.mask_submc1.endFill();*/

		}
}