Spaces:
No application file
No application file
File size: 11,060 Bytes
b7731cd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 |
# Copyright 2020 by Michiel de Hoon
#
# This file is part of the Biopython distribution and governed by your
# choice of the "Biopython License Agreement" or the "BSD 3-Clause License".
# Please see the LICENSE file that should have been included as part of this
# package.
"""Bio.SeqIO support for UCSC's "twoBit" (.2bit) file format.
This parser reads the index stored in the twoBit file, as well as the masked
regions and the N's for each sequence. It also creates sequence data objects
(_TwoBitSequenceData objects), which support only two methods: __len__ and
__getitem__. The former will return the length of the sequence, while the
latter returns the sequence (as a bytes object) for the requested region.
Using the information in the index, the __getitem__ method calculates the file
position at which the requested region starts, and only reads the requested
sequence region. Note that the full sequence of a record is loaded only if
specifically requested, making the parser memory-efficient.
The TwoBitIterator object implements the __getitem__, keys, and __len__
methods that allow it to be used as a dictionary.
"""
# The .2bit file format is defined by UCSC as follows
# (see http://genome.ucsc.edu/FAQ/FAQformat.html#format7):
#
#
# A .2bit file stores multiple DNA sequences (up to 4 Gb total) in a compact
# randomly-accessible format. The file contains masking information as well
# as the DNA itself.
#
# The file begins with a 16-byte header containing the following fields:
#
# signature - the number 0x1A412743 in the architecture of the machine that
# created the file
# version - zero for now. Readers should abort if they see a version number
# higher than 0
# sequenceCount - the number of sequences in the file
# reserved - always zero for now
#
# All fields are 32 bits unless noted. If the signature value is not as
# given, the reader program should byte-swap the signature and check if the
# swapped version matches. If so, all multiple-byte entities in the file
# will have to be byte-swapped. This enables these binary files to be used
# unchanged on different architectures.
#
# The header is followed by a file index, which contains one entry for each
# sequence. Each index entry contains three fields:
#
# nameSize - a byte containing the length of the name field
# name - the sequence name itself (in ASCII-compatible byte string), of
# variable length depending on nameSize
# offset - the 32-bit offset of the sequence data relative to the start of
# the file, not aligned to any 4-byte padding boundary
#
# The index is followed by the sequence records, which contain nine fields:
#
# dnaSize - number of bases of DNA in the sequence
# nBlockCount - the number of blocks of Ns in the file (representing unknown
# sequence)
# nBlockStarts - an array of length nBlockCount of 32 bit integers
# indicating the (0-based) starting position of a block of Ns
# nBlockSizes - an array of length nBlockCount of 32 bit integers indicating
# the length of a block of Ns
# maskBlockCount - the number of masked (lower-case) blocks
# maskBlockStarts - an array of length maskBlockCount of 32 bit integers
# indicating the (0-based) starting position of a masked block
# maskBlockSizes - an array of length maskBlockCount of 32 bit integers
# indicating the length of a masked block
# reserved - always zero for now
# packedDna - the DNA packed to two bits per base, represented as so:
# T - 00, C - 01, A - 10, G - 11. The first base is in the most
# significant 2-bit byte; the last base is in the least significan
# 2 bits. For example, the sequence TCAG is represented as 00011011.
import numpy
from Bio.Seq import Seq
from Bio.Seq import SequenceDataAbstractBaseClass
from Bio.SeqRecord import SeqRecord
from . import _twoBitIO
from .Interfaces import SequenceIterator
class _TwoBitSequenceData(SequenceDataAbstractBaseClass):
"""Stores information needed to retrieve sequence data from a .2bit file (PRIVATE).
Objects of this class store the file position at which the sequence data
start, the sequence length, and the start and end position of unknown (N)
and masked (lowercase) letters in the sequence.
Only two methods are provided: __len__ and __getitem__. The former will
return the length of the sequence, while the latter returns the sequence
(as a bytes object) for the requested region. The full sequence of a record
is loaded only if explicitly requested.
"""
__slots__ = ("stream", "offset", "length", "nBlocks", "maskBlocks")
def __init__(self, stream, offset, length):
"""Initialize the file stream and file position of the sequence data."""
self.stream = stream
self.offset = offset
self.length = length
super().__init__()
def __getitem__(self, key):
"""Return the sequence contents (as a bytes object) for the requested region."""
length = self.length
if isinstance(key, slice):
start, end, step = key.indices(length)
size = len(range(start, end, step))
if size == 0:
return b""
else:
if key < 0:
key += length
if key < 0:
raise IndexError("index out of range")
start = key
end = key + 1
step = 1
size = 1
byteStart = start // 4
byteEnd = (end + 3) // 4
byteSize = byteEnd - byteStart
stream = self.stream
try:
stream.seek(self.offset + byteStart)
except ValueError as exception:
if str(exception) == "seek of closed file":
raise ValueError("cannot retrieve sequence: file is closed") from None
raise
data = numpy.fromfile(stream, dtype="uint8", count=byteSize)
sequence = _twoBitIO.convert(
data, start, end, step, self.nBlocks, self.maskBlocks
)
if isinstance(key, slice):
return sequence
else: # single nucleotide
return ord(sequence)
def __len__(self):
"""Get the sequence length."""
return self.length
def upper(self):
"""Remove the sequence mask."""
data = _TwoBitSequenceData(self.stream, self.offset, self.length)
data.nBlocks = self.nBlocks[:, :]
data.maskBlocks = numpy.empty((0, 2), dtype="uint32")
return data
def lower(self):
"""Extend the sequence mask to the full sequence."""
data = _TwoBitSequenceData(self.stream, self.offset, self.length)
data.nBlocks = self.nBlocks[:, :]
data.maskBlocks = numpy.array([[0, self.length]], dtype="uint32")
return data
class TwoBitIterator(SequenceIterator):
"""Parser for UCSC twoBit (.2bit) files."""
def __init__(self, source):
"""Read the file index."""
super().__init__(source, mode="b", fmt="twoBit")
# wait to close the file until the TwoBitIterator goes out of scope:
self.should_close_stream = False
stream = self.stream
data = stream.read(4)
if not data:
raise ValueError("Empty file.")
byteorders = ("little", "big")
dtypes = ("<u4", ">u4")
for byteorder, dtype in zip(byteorders, dtypes):
signature = int.from_bytes(data, byteorder)
if signature == 0x1A412743:
break
else:
raise ValueError("Unknown signature")
self.byteorder = byteorder
data = stream.read(4)
version = int.from_bytes(data, byteorder, signed=False)
if version == 1:
raise ValueError(
"version-1 twoBit files with 64-bit offsets for index are currently not supported"
)
if version != 0:
raise ValueError("Found unexpected file version %u; aborting" % version)
data = stream.read(4)
sequenceCount = int.from_bytes(data, byteorder, signed=False)
data = stream.read(4)
reserved = int.from_bytes(data, byteorder, signed=False)
if reserved != 0:
raise ValueError("Found non-zero reserved field; aborting")
sequences = {}
for i in range(sequenceCount):
data = stream.read(1)
nameSize = int.from_bytes(data, byteorder, signed=False)
data = stream.read(nameSize)
name = data.decode("ASCII")
data = stream.read(4)
offset = int.from_bytes(data, byteorder, signed=False)
sequences[name] = (stream, offset)
self.sequences = sequences
for name, (stream, offset) in sequences.items():
stream.seek(offset)
data = stream.read(4)
dnaSize = int.from_bytes(data, byteorder, signed=False)
sequence = _TwoBitSequenceData(stream, offset, dnaSize)
data = stream.read(4)
nBlockCount = int.from_bytes(data, byteorder, signed=False)
nBlockStarts = numpy.fromfile(stream, dtype=dtype, count=nBlockCount)
nBlockSizes = numpy.fromfile(stream, dtype=dtype, count=nBlockCount)
sequence.nBlocks = numpy.empty((nBlockCount, 2), dtype="uint32")
sequence.nBlocks[:, 0] = nBlockStarts
sequence.nBlocks[:, 1] = nBlockStarts + nBlockSizes
data = stream.read(4)
maskBlockCount = int.from_bytes(data, byteorder, signed=False)
maskBlockStarts = numpy.fromfile(stream, dtype=dtype, count=maskBlockCount)
maskBlockSizes = numpy.fromfile(stream, dtype=dtype, count=maskBlockCount)
sequence.maskBlocks = numpy.empty((maskBlockCount, 2), dtype="uint32")
sequence.maskBlocks[:, 0] = maskBlockStarts
sequence.maskBlocks[:, 1] = maskBlockStarts + maskBlockSizes
data = stream.read(4)
reserved = int.from_bytes(data, byteorder, signed=False)
if reserved != 0:
raise ValueError("Found non-zero reserved field %u" % reserved)
sequence.offset = stream.tell()
sequences[name] = sequence
def parse(self, stream):
"""Iterate over the sequences in the file."""
for name, sequence in self.sequences.items():
sequence = Seq(sequence)
record = SeqRecord(sequence, id=name)
yield record
def __getitem__(self, name):
"""Return sequence associated with given name as a SeqRecord object."""
try:
sequence = self.sequences[name]
except ValueError:
raise KeyError(name) from None
sequence = Seq(sequence)
return SeqRecord(sequence, id=name)
def keys(self):
"""Return a list with the names of the sequences in the file."""
return self.sequences.keys()
def __len__(self):
"""Return number of sequences."""
return len(self.sequences)
|